Correcting the pims_split metadata and dag files to iterate over the splits

shared/workspaces/alembic/versions/762c98a8adf1_pims_split_quicklook_corrections.py

+"""pims split quicklook corrections
+
+Revision ID: 762c98a8adf1
+Revises: e00812d93608
+Create Date: 2023-04-14 12:25:42.235329
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '762c98a8adf1'
+down_revision = 'e00812d93608'
+branch_labels = None
+depends_on = None
+
+
+"""
+Iterating over {{#splits}} to make the "splits" array which has the format:
+"splits": [
+    {
+        "split_dir": {{split_dir}},
+        "quicklook_url": {{quicklook_url}}
+    },
+]
+"""
+old_metadata = """{"systemId": "{{request_id}}", "fileSetIds": ["{{sdmId}}", "{{calSdmId}}"], "creationTime": "{{created_at}}", "workflowName": "pims_split",  "productLocator": "{{product_locator}}", "calProductLocator": "{{cal_locator}}", "destinationDirectory": "{{root_directory}}/{{relative_path}}", "token": "{{token}}", "splits": ["{{split_dir}}", "{{quicklook_url}}"], "casaHome": "{{casaHome}}", "data_location": "{{data_location}}", "vlass_product": "{{vlass_product}}", "existing_restore": "{{existing_restore}}", "need_project_metadata": "{{need_project_metadata}}"}"""
+
+new_metadata = """{"systemId": "{{request_id}}", "fileSetIds": ["{{sdmId}}", "{{calSdmId}}"], "creationTime": "{{created_at}}", "workflowName": "pims_split",  "productLocator": "{{product_locator}}", "calProductLocator": "{{cal_locator}}", "destinationDirectory": "{{root_directory}}/{{relative_path}}", "token": "{{token}}", "splits": [{{#splits}}{"split_dir": "{{split_dir}}", "quicklook_url": "{{quicklook_url}}"},{{/splits}}], "casaHome": "{{casaHome}}", "data_location": "{{data_location}}", "vlass_product": "{{vlass_product}}", "existing_restore": "{{existing_restore}}", "need_project_metadata": "{{need_project_metadata}}"}"""
+
+
+# It should be safe if we pass an empty quicklook_url to the condor file since there's an empty string check in split.sh
+old_dag = """JOB RESTORE restore.condor
+VARS RESTORE jobname="$(JOB)"
+
+{{#splits}}
+JOB {{#make_condor_jobname}}{{split_dir}}{{/make_condor_jobname}} split.condor
+VARS {{#make_condor_jobname}}{{split_dir}}{{/make_condor_jobname}} jobname="$(JOB)" split_dir="{{split_dir}}" {{#quicklook_url}}quicklook_url="{{quicklook_url}}"{{/quicklook_url}}
+
+{{/splits}}
+
+JOB FINISH write_finished_file.condor
+VARS FINISH jobname="$(JOB)"
+
+PARENT RESTORE CHILD {{#splits}}{{#make_condor_jobname}}{{split_dir}}{{/make_condor_jobname}} {{/splits}}
+PARENT {{#splits}}{{#make_condor_jobname}}{{split_dir}}{{/make_condor_jobname}} {{/splits}} CHILD FINISH
+"""
+
+new_dag = """JOB RESTORE restore.condor
+VARS RESTORE jobname="$(JOB)"
+
+{{#splits}}
+JOB {{#make_condor_jobname}}{{split_dir}}{{/make_condor_jobname}} split.condor
+VARS {{#make_condor_jobname}}{{split_dir}}{{/make_condor_jobname}} jobname="$(JOB)" split_dir="{{split_dir}}" quicklook_url="{{quicklook_url}}"
+
+{{/splits}}
+
+JOB FINISH write_finished_file.condor
+VARS FINISH jobname="$(JOB)"
+
+PARENT RESTORE CHILD {{#splits}}{{#make_condor_jobname}}{{split_dir}}{{/make_condor_jobname}} {{/splits}}
+PARENT {{#splits}}{{#make_condor_jobname}}{{split_dir}}{{/make_condor_jobname}} {{/splits}} CHILD FINISH
+"""
+
+
+def upgrade():
+    op.execute(f"""
+        UPDATE workflow_templates
+        SET content = E'{new_metadata}'
+        WHERE workflow_name = 'pims_split' AND filename = 'metadata.json' 
+    """)
+
+    op.execute(f"""
+        UPDATE workflow_templates
+        SET content = E'{new_dag}'
+        WHERE workflow_name = 'pims_split' AND filename = 'pims_split.dag' 
+    """)
+
+
+def downgrade():
+    op.execute(f"""
+        UPDATE workflow_templates
+        SET content = E'{old_metadata}'
+        WHERE workflow_name = 'pims_split' AND filename = 'metadata.json' 
+    """)
+
+    op.execute(f"""
+            UPDATE workflow_templates
+            SET content = E'{old_dag}'
+            WHERE workflow_name = 'pims_split' AND filename = 'pims_split.dag' 
+    """)