diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
index eee79d0eaa113c94975081982c0eeb1b9a153224..88ca1f8c3c5b9b103a6a1bc40df49ffb3040629e 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/ingestion_manifest.py
@@ -162,7 +162,7 @@ class IngestionManifest(ManifestComponentIF):
:return:
"""
- me_dict = self.to_json()
+ me_dict = self.to_dict()
output_path = self.staging_source_dir / MANIFEST_FILENAME
to_write = json.dumps(me_dict, indent=4)
@@ -171,9 +171,9 @@ class IngestionManifest(ManifestComponentIF):
return output_path
- def to_json(self) -> JSON:
+ def to_dict(self) -> JSON:
"""
- Turn this object into a JSON string suitable for writing to a file
+ Turn this maifest into a Dictionary suitable for converting to JSON
:return:
"""
@@ -181,15 +181,15 @@ class IngestionManifest(ManifestComponentIF):
me_dict = dict(self.__dict__)
to_return = {
- IngestionManifestKey.PARAMETERS.value: self.parameters.to_json(),
- IngestionManifestKey.OUTPUT_GROUP.value: me_dict[IngestionManifestKey.OUTPUT_GROUP.value].to_json(),
+ IngestionManifestKey.PARAMETERS.value: self.parameters.to_dict(),
+ IngestionManifestKey.OUTPUT_GROUP.value: me_dict[IngestionManifestKey.OUTPUT_GROUP.value].to_dict(),
}
# Ingestion manifests with empty input groups can cause errors on ingest
if len(self.input_group.science_products) > 0:
to_return[IngestionManifestKey.INPUT_GROUP.value] = me_dict[
IngestionManifestKey.INPUT_GROUP.value
- ].to_json()
+ ].to_dict()
return to_return
@@ -227,7 +227,6 @@ class IngestionManifestBuilder:
ScienceProductType.VLASS_SECI,
ScienceProductType.EXEC_BLOCK,
]:
-
raise NotImplementedError(f"Don't know yet how to build a {self.sp_type.value} manifest")
if locator is not None:
diff --git a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py
index 9b6c5d1b0d1f9c86586c0d7b832fa66572273e80..6e53946d4ff78d1c3c331a4ee255a903da049b2f 100644
--- a/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py
+++ b/apps/cli/executables/pexable/ingest_envoy/ingest_envoy/manifest_components.py
@@ -67,14 +67,22 @@ class ManifestComponentIF(abc.ABC):
def __eq__(self, other):
return NotImplemented
+ # @abc.abstractmethod
+ # def to_json(self) -> JSON:
+ # """
+ # JSON-ify this object
+ #
+ # :return: json.load()-able string
+ # """
+ # return NotImplemented
+
@abc.abstractmethod
- def to_json(self) -> JSON:
+ def to_dict(self) -> Dict:
"""
- JSON-ify this object
+ Build a JSON-ifyable dictionary
- :return: json.load()-able string
+ :return: json.dumps()-able dictionary object
"""
- return NotImplemented
class InputScienceProduct(ManifestComponentIF):
@@ -87,7 +95,7 @@ class InputScienceProduct(ManifestComponentIF):
if isinstance(other, InputScienceProduct):
return other.locator == self.locator
- def to_json(self) -> JSON:
+ def to_dict(self) -> JSON:
"""
Turn me into a json-ifiable dict
@@ -108,7 +116,7 @@ class InputGroup(ManifestComponentIF):
return False
- def to_json(self) -> JSON:
+ def to_dict(self) -> JSON:
"""
Turn me into a json-ifiable dict
@@ -117,7 +125,7 @@ class InputGroup(ManifestComponentIF):
me_dict = dict(self.__dict__)
sps = me_dict[IngestionManifestKey.SCIENCE_PRODUCTS.value]
- sps = [sp.to_json() for sp in sps]
+ sps = [sp.to_dict() for sp in sps]
if len(sps) == 0:
return {}
@@ -165,18 +173,22 @@ class ManifestParameters(ManifestComponentIF):
return False
- def to_json(self) -> JSON:
+ def to_dict(self) -> JSON:
+ """
+ form dictionary object for this manifest section
+ :return:
+ """
json_dict = {
ParamsKey.TELESCOPE.value: self.telescope.value,
# The ingestion manifest must have "true" and "false"
# rather than "True" and "False"
- ParamsKey.REINGEST.value: "true" if self.reingest else "false",
- ParamsKey.NGAS_INGEST.value: "true" if self.ngas_ingest else "false",
+ ParamsKey.REINGEST.value: self.reingest,
+ ParamsKey.NGAS_INGEST.value: self.ngas_ingest,
ParamsKey.INGESTION_PATH.value: str(self.staging_source_dir),
}
if hasattr(self, "calibrate"):
- json_dict[ParamsKey.CALIBRATE.value] = "true" if self.calibrate else "false"
+ json_dict[ParamsKey.CALIBRATE.value] = self.calibrate
if self.additional_metadata:
json_dict[ParamsKey.ADDITIONAL_METADATA.value] = self.additional_metadata.filename
@@ -214,7 +226,7 @@ class AncillaryProduct(ManifestComponentIF):
def __str__(self):
return f"{self.filename}: {self.type.value}"
- def to_json(self) -> JSON:
+ def to_dict(self) -> JSON:
"""
Turn me into a json-ifiable dict
@@ -256,10 +268,10 @@ class OutputScienceProduct(ManifestComponentIF):
def __str__(self):
return f"{Path(self.filename).name}: {self.type.value}, {len(self.ancillary_products)} ancillary products"
- def to_json(self) -> JSON:
+ def to_dict(self) -> JSON:
json_dict = {"type": self.type.value, "filename": self.filename}
if self.ancillary_products:
- aps_json = [ap.to_json() for ap in self.ancillary_products]
+ aps_json = [ap.to_dict() for ap in self.ancillary_products]
json_dict[IngestionManifestKey.ANCILLARY_PRODUCTS.value] = aps_json
return json_dict
@@ -283,7 +295,7 @@ class OutputGroup(ManifestComponentIF):
return False
- def to_json(self) -> JSON:
+ def to_dict(self) -> JSON:
"""
Turn me into a json-ifiable dict
@@ -292,12 +304,12 @@ class OutputGroup(ManifestComponentIF):
me_dict = dict(self.__dict__)
sps = me_dict[IngestionManifestKey.SCIENCE_PRODUCTS.value]
- sps = [sp.to_json() for sp in sps]
+ sps = [sp.to_dict() for sp in sps]
me_dict[IngestionManifestKey.SCIENCE_PRODUCTS.value] = sps
aps = me_dict[IngestionManifestKey.ANCILLARY_PRODUCTS.value]
if aps:
- ap_jsons = [ap.to_json() for ap in aps]
+ ap_jsons = [ap.to_dict() for ap in aps]
me_dict[IngestionManifestKey.ANCILLARY_PRODUCTS.value] = ap_jsons
return me_dict
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py b/apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py
index 64e2b7df0a1fca3dfea3e0257deb7e2e4a0756bb..3eb8025d5645d89e260b3021667a898582813286 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_evla_cal_manifest.py
@@ -168,7 +168,7 @@ def test_params_json_well_formed():
staging_source_dir=Path("/home/mchammer/evla/parallel-prod"),
)
- params_json = params.to_json()
+ params_json = params.to_dict()
# if we can dump it, it's good
json.dumps(params_json)
@@ -186,7 +186,7 @@ def test_input_sp_well_formed():
}
sp_in = InputScienceProduct(locator=locator)
- assert sp_in.to_json() == sp_dict
+ assert sp_in.to_dict() == sp_dict
def test_input_group_well_formed():
@@ -199,7 +199,7 @@ def test_input_group_well_formed():
sp2 = InputScienceProduct(locator="uid://evla/execblock/mint_oreo_omg_omg")
input_group = InputGroup(science_products=[sp1, sp2])
- assert IngestionManifestKey.SCIENCE_PRODUCTS.value in input_group.to_json().keys()
+ assert IngestionManifestKey.SCIENCE_PRODUCTS.value in input_group.to_dict().keys()
sps_found = input_group.science_products
assert len(sps_found) == 2
@@ -215,7 +215,7 @@ def test_ancillary_product_well_formed():
"""
ap1 = AncillaryProduct(type=AncillaryProductType.LOG_TYPE, filename="without_feathers.tar")
expected = {"type": AncillaryProductType.LOG_TYPE.value, "filename": ap1.filename}
- actual = ap1.to_json()
+ actual = ap1.to_dict()
assert actual == expected
@@ -239,10 +239,10 @@ def test_output_group_well_formed():
assert "/" not in ancillary_product.filename
expected_json = {
- IngestionManifestKey.SCIENCE_PRODUCTS.value: [osp.to_json()],
- IngestionManifestKey.ANCILLARY_PRODUCTS.value: [ap1.to_json(), ap2.to_json()],
+ IngestionManifestKey.SCIENCE_PRODUCTS.value: [osp.to_dict()],
+ IngestionManifestKey.ANCILLARY_PRODUCTS.value: [ap1.to_dict(), ap2.to_dict()],
}
- actual_json = opg.to_json()
+ actual_json = opg.to_dict()
assert actual_json == expected_json
@@ -334,15 +334,15 @@ def test_evla_cal_manifest_matches_example(ingest_path: Path):
actual_json[IngestionManifestKey.PARAMETERS.value][IngestionManifestKey.INGESTION_PATH.value] = expected_dir_name
expected_params = expected_json["parameters"]
- actual_params = manifest.parameters.to_json()
+ actual_params = manifest.parameters.to_dict()
- expected_reingest = "true" if expected_params["reingest"] else "false"
+ expected_reingest = expected_params["reingest"]
assert actual_params["reingest"] == expected_reingest
- expected_ngas_ingest = "true" if expected_params["ngas_ingest"] else "false"
+ expected_ngas_ingest = expected_params["ngas_ingest"]
assert actual_params["ngas_ingest"] == expected_ngas_ingest
assert "calibrate" not in actual_params.keys()
- assert manifest.input_group.to_json() == expected_json["input_group"]
+ assert manifest.input_group.to_dict() == expected_json["input_group"]
expected_outgroup = expected_json["output_group"]
expected_osp = expected_outgroup["science_products"]
@@ -438,15 +438,15 @@ def test_evla_cal_final_manifest_matches_example(ingest_path: Path):
actual_json[IngestionManifestKey.PARAMETERS.value][IngestionManifestKey.INGESTION_PATH.value] = expected_dir_name
expected_params = expected_json["parameters"]
- actual_params = manifest.parameters.to_json()
+ actual_params = manifest.parameters.to_dict()
- expected_reingest = "true" if expected_params["reingest"] else "false"
+ expected_reingest = expected_params["reingest"]
assert actual_params["reingest"] == expected_reingest
- expected_ngas_ingest = "true" if expected_params["ngas_ingest"] else "false"
+ expected_ngas_ingest = expected_params["ngas_ingest"]
assert actual_params["ngas_ingest"] == expected_ngas_ingest
assert "calibrate" not in actual_params.keys()
- assert manifest.input_group.to_json() == expected_json["input_group"]
+ assert manifest.input_group.to_dict() == expected_json["input_group"]
expected_outgroup = expected_json["output_group"]
expected_osp = expected_outgroup["science_products"]
diff --git a/apps/cli/executables/pexable/ingest_envoy/test/test_img_manifest_example.py b/apps/cli/executables/pexable/ingest_envoy/test/test_img_manifest_example.py
index 3c56f7cef6d3b2f2174fe4aae98f91717ab4a81a..1bdac66167c96ff96a16d6ee202e298d1268565f 100644
--- a/apps/cli/executables/pexable/ingest_envoy/test/test_img_manifest_example.py
+++ b/apps/cli/executables/pexable/ingest_envoy/test/test_img_manifest_example.py
@@ -79,7 +79,7 @@ def test_addl_metadata_not_at_bottom_of_manifest(ingest_path: Path):
"""
populate_fake_tmpx_ratuqh_ingest_path(ingest_path)
manifest, _ = build_tmpx_ratuqh_image_manifest(ingest_path)
- mf_json = manifest.to_json()
+ mf_json = manifest.to_dict()
keys = ["parameters", "input_group", "output_group"]
assert len(mf_json) == len(keys)
for key in keys:
@@ -98,7 +98,7 @@ def test_manifest_picks_up_pip_artifact(ingest_path: Path):
maybe_pips = [file for file in ingest_path.glob(AncillaryProductType.PIPELINE_ARTIFACTS.value + "*.tar")]
pip_artie = maybe_pips[0]
- mf_json = manifest.to_json()
+ mf_json = manifest.to_dict()
og_json = mf_json[IngestionManifestKey.OUTPUT_GROUP.value]
ap_jsons = og_json[IngestionManifestKey.ANCILLARY_PRODUCTS.value]
@@ -189,7 +189,7 @@ def test_ancillary_products_rendered_correctly(ingest_path: Path):
ap_data = []
for ap in aps:
- ap_data.append({"filename": ap.filename, "json": ap.to_json()})
+ ap_data.append({"filename": ap.filename, "json": ap.to_dict()})
assert len(ap_data) == len(aps)
# make sure all the ancillary products were created...
@@ -228,7 +228,7 @@ def test_output_science_products_rendered_correctly(ingest_path: Path):
"""
populate_fake_tmpx_ratuqh_ingest_path(ingest_path)
manifest, _ = build_tmpx_ratuqh_image_manifest(ingest_path)
- mf_json = manifest.to_json()
+ mf_json = manifest.to_dict()
og_json = mf_json[IngestionManifestKey.OUTPUT_GROUP.value]
print(og_json)