diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py index 5c074f8fa05e20689e7e07f8c68d161122039dd2..282cea855b9ab1e94f1028bdda5fb196ba7eacc6 100644 --- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py +++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py @@ -170,9 +170,6 @@ class IngestionManifestBuilder: :return: the ingestion manifest and the file containing its JSON """ - # # create any other ingestion files needed for this type of ingestion - # self._find_additional_ingestion_files() - if self.sp_type == ScienceProductType.EVLA_CAL: return self._build_evla_cal_manifest() @@ -192,9 +189,9 @@ class IngestionManifestBuilder: # We can't create the ingestion artifacts tar quite yet, # because it will contain the yet-to-be-written manifest itself # (required for ingestion, evidently) - artifacts_file = self.staging_source_dir / self._build_artifacts_filename() + artifacts_filename = self._build_artifacts_filename() artifacts_ap = AncillaryProduct( - AncillaryProductType.INGESTION_ARTIFACTS, filename=str(artifacts_file) + AncillaryProductType.INGESTION_ARTIFACTS, filename=artifacts_filename ) if not manifest.output_group.ancillary_products: @@ -206,6 +203,7 @@ class IngestionManifestBuilder: manifest.output_group.ancillary_products.append(artifacts_ap) manifest_file = manifest.write() + artifacts_file = self.staging_source_dir / self._build_artifacts_filename() self.write_ingestion_artifacts_tar(artifacts_file) return manifest, manifest_file @@ -262,7 +260,7 @@ class IngestionManifestBuilder: tars_found = find_output_tars(self.files_found, self.staging_source_dir) for file in tars_found: - sci_prod = OutputScienceProduct(type=self.sp_type, filename=str(file)) + sci_prod = OutputScienceProduct(type=self.sp_type, filename=file.name) return OutputGroup(science_products=[sci_prod]) diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_if.py b/apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py similarity index 95% rename from apps/cli/executables/pexable/ingest_envoy/test/test_manifest_if.py rename to apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py index 33ddeab9db78ff0fb13d4c39cf37e5bcda781b75..ef74962afd429a5b4ae6c5ed6587ef63bf04cb18 100644 --- a/apps/cli/executables/pexable/ingest_envoy/test/test_manifest_if.py +++ b/apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py @@ -50,10 +50,9 @@ logger.addHandler(logging.StreamHandler(sys.stdout)) FAKE_LOCATOR = "uid://evla/calibration/doo-wah-ditty-ditty-af123" -@pytest.mark.skip("TODO: broken temporarily, pending fix to output group creation") def test_filters_cal_input_files(ingest_path: Path): """ - We'll be getting calibration/image/eb, etc. science products from a directory under + We'll be getting science products from a directory under /lustre/aoc/cluster/pipeline/{CAPO_PROFILE}/workspaces/staging Make sure we take -only- the files to be ingested. @@ -80,8 +79,9 @@ def test_filters_cal_input_files(ingest_path: Path): assert len(input_group.science_products) == 1 output_group = manifest.output_group + assert len(output_group.science_products) == 1 - assert len(output_group.ancillary_products) == 1 + assert len(output_group.ancillary_products) == 2 for product in output_group.ancillary_products: if product.filename not in EVLA_CAL_INPUT_FILENAMES: assert product.filename.startswith( @@ -210,11 +210,18 @@ def test_output_group_well_formed(): type=AncillaryProductType.PIPELINE_ARTIFACTS, filename="with_feathers.tar" ) + opg = OutputGroup(science_products=[osp], ancillary_products=[ap1, ap2]) + + # all filenames in the manifest should be bare filenames, not full paths + for science_product in opg.science_products: + assert "/" not in science_product.filename + for ancillary_product in opg.ancillary_products: + assert "/" not in ancillary_product.filename + expected_json = { IngestionManifestKey.SCIENCE_PRODUCTS.value: [osp.to_json()], IngestionManifestKey.ANCILLARY_PRODUCTS.value: [ap1.to_json(), ap2.to_json()], } - opg = OutputGroup(science_products=[osp], ancillary_products=[ap1, ap2]) actual_json = opg.to_json() assert actual_json == expected_json @@ -283,7 +290,7 @@ def test_evla_cal_manifest_matches_example(ingest_path: Path): IngestionManifestKey.INGESTION_PATH.value ] = expected_dir_name - # TODO 2021-08-02pm NEXT MR : get this to work (and don't worry about the JSON per se) + # TODO 2021-08-03am NEXT MR : get this to work (and don't worry about the JSON per se) # expected_params = expected_json["parameters"] # actual_params = manifest.parameters.to_json() # actual_params["ingestion_path"] = expected_params["ingestion_path"]