Skip to content
Snippets Groups Projects

WS-561: Capability definition UI simplification

Merged Nathan Hertz requested to merge WS-461-capability-step-sequence-overhaul into main
Files
14
@@ -26,6 +26,7 @@ from ingest_envoy.manifest_components import (
OutputGroup,
MANIFEST_FILENAME,
ParamsKey,
WEBLOG_FILENAME,
)
from ingest_envoy.schema import AbstractTextFile
from ingest_envoy.std_img_manifest_utils import ImageIngestionProductsFinder
@@ -84,21 +85,30 @@ class ManifestIF(ManifestComponentIF):
params = ManifestParameters(
telescope=self.telescope,
reingest=False,
ngas_ingest=False,
ngas_ingest=True,
calibrate=False,
staging_source_dir=self.staging_source_dir,
additional_metadata=additional_metadata,
)
else:
self.additional_metadata = additional_metadata
elif self.sp_type == ScienceProductType.IMAGE:
params = ManifestParameters(
telescope=self.telescope,
reingest=False,
ngas_ingest=False,
ngas_ingest=True,
calibrate=False,
staging_source_dir=self.staging_source_dir,
)
self.additional_metadata = additional_metadata
else:
params = ManifestParameters(
telescope=self.telescope,
reingest=False,
ngas_ingest=True,
staging_source_dir=self.staging_source_dir,
)
return params
@abc.abstractmethod
@@ -135,6 +145,8 @@ class ManifestIF(ManifestComponentIF):
class IngestionManifestBuilder:
"""Builds ingestion manifest and associated files from files in ingestion_path"""
# TODO?: We are making assumptions about the values of "reingest", "ngas_ingest",
# and "calibrate"; is this valid?
def __init__(
self,
staging_source_dir: Path,
@@ -169,9 +181,6 @@ class IngestionManifestBuilder:
:return: the ingestion manifest and the file containing its JSON
"""
# # create any other ingestion files needed for this type of ingestion
# self._find_additional_ingestion_files()
if self.sp_type == ScienceProductType.EVLA_CAL:
return self._build_evla_cal_manifest()
@@ -187,13 +196,26 @@ class IngestionManifestBuilder:
input_group=self._build_input_group(),
output_group=self._build_evla_cal_output_group(),
)
manifest_file = manifest.write()
artifacts_tar = self.write_ingestion_artifacts_tar()
# We can't create the ingestion artifacts tar quite yet,
# because it will contain the yet-to-be-written manifest itself
# (required for ingestion, evidently)
artifacts_filename = self._build_artifacts_filename()
artifacts_ap = AncillaryProduct(
AncillaryProductType.INGESTION_ARTIFACTS, filename=artifacts_tar.name
AncillaryProductType.INGESTION_ARTIFACTS, filename=artifacts_filename
)
manifest.output_group.ancillary_products = [artifacts_ap]
if not manifest.output_group.ancillary_products:
manifest.output_group.ancillary_products = []
weblog_ap = AncillaryProduct(
type=AncillaryProductType.PIPELINE_WEBLOG_TYPE, filename=WEBLOG_FILENAME
)
manifest.output_group.ancillary_products.append(weblog_ap)
manifest.output_group.ancillary_products.append(artifacts_ap)
manifest_file = manifest.write()
artifacts_file = self.staging_source_dir / artifacts_filename
self.write_ingestion_artifacts_tar(artifacts_file)
return manifest, manifest_file
@@ -215,11 +237,12 @@ class IngestionManifestBuilder:
manifest_file = manifest.write()
artifacts_tar = self.write_ingestion_artifacts_tar()
artifacts_file = self.staging_source_dir / self._build_artifacts_filename()
artifacts_ap = AncillaryProduct(
type=AncillaryProductType.INGESTION_ARTIFACTS, filename=artifacts_tar.name
type=AncillaryProductType.INGESTION_ARTIFACTS, filename=str(artifacts_file)
)
manifest.output_group.ancillary_products.append(artifacts_ap)
self.write_ingestion_artifacts_tar(artifacts_file)
return manifest, manifest_file
@@ -248,7 +271,7 @@ class IngestionManifestBuilder:
tars_found = find_output_tars(self.files_found, self.staging_source_dir)
for file in tars_found:
sci_prod = OutputScienceProduct(type=self.sp_type, filename=str(file))
sci_prod = OutputScienceProduct(type=self.sp_type, filename=file.name)
return OutputGroup(science_products=[sci_prod])
@@ -267,7 +290,7 @@ class IngestionManifestBuilder:
return OutputGroup(science_products=science_products, ancillary_products=ancillary_products)
@staticmethod
def build_artifacts_filename() -> str:
def _build_artifacts_filename() -> str:
"""
Build unique manifest filename in standard format.
@@ -277,7 +300,7 @@ class IngestionManifestBuilder:
timestamp = format_timestamp(current_time)
return f"{INGESTION_ARTIFACTS_NAME}{timestamp}{TARFILE_EXT}"
def write_ingestion_artifacts_tar(self) -> tarfile.TarFile:
def write_ingestion_artifacts_tar(self, artifacts_path: Path) -> tarfile.TarFile:
"""
Take the list of files and build a tar for inclusion into the archive.
This happens in the staging area for ingestion.
@@ -286,13 +309,14 @@ class IngestionManifestBuilder:
:return: a .tar archive of the ingestion artifacts
"""
manifest_file = find_manifest(self.staging_source_dir)
ing_tar = self.staging_source_dir / self.build_artifacts_filename()
with tarfile.open(ing_tar, "w") as ingestion_artifacts_tar:
with tarfile.open(artifacts_path, "w") as ingestion_artifacts_tar:
for file in self.files_found:
ingestion_artifacts_tar.add(file)
if self.sp_type == ScienceProductType.IMAGE:
ingestion_artifacts_tar.add(file)
# include the manifest
# The manifest file itself is considered an ingestion artifact.
# (It's turtles all the way down.)
manifest_file = self.staging_source_dir / MANIFEST_FILENAME
ingestion_artifacts_tar.add(manifest_file)
return ingestion_artifacts_tar
Loading