Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
workspaces
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
ssa
workspaces
Commits
8e3ead95
Commit
8e3ead95
authored
8 months ago
by
Sam Kagan
Browse files
Options
Downloads
Patches
Plain Diff
Drafted RestoreProductFinder for deliver
parent
7fb2cdcf
No related branches found
Branches containing commit
No related tags found
No related merge requests found
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
apps/cli/executables/pexable/deliver/delivery/finder.py
+46
-4
46 additions, 4 deletions
apps/cli/executables/pexable/deliver/delivery/finder.py
apps/cli/executables/pexable/deliver/delivery/products.py
+13
-0
13 additions, 0 deletions
apps/cli/executables/pexable/deliver/delivery/products.py
with
59 additions
and
4 deletions
apps/cli/executables/pexable/deliver/delivery/finder.py
+
46
−
4
View file @
8e3ead95
...
...
@@ -20,19 +20,23 @@
# P R O D U C T F I N D I N G
#
# -------------------------------------------------------------------------
import
json
import
abc
import
json
import
pathlib
import
re
from
typing
import
Iterator
,
List
from
.products
import
SpooledProduct
,
ArchiveProduct
import
pendulum
from
pendulum.datetime
import
DateTime
from
.products
import
ArchiveProduct
,
ProductMetadata
,
RestoreProduct
,
SpooledProduct
class
ProductFinder
(
abc
.
ABC
):
"""
Locates products for the delivery to deliver
"""
@property
@abc.abstractmethod
def
projects
(
self
)
->
List
[
str
]:
...
...
@@ -104,8 +108,46 @@ class JsonProductFinder(ProductFinder):
Return a list of all the projects we
'
re delivering for right now.
:return:
"""
return
list
(
set
(
p
[
'
project_code
'
]
for
p
in
self
.
products
.
values
()))
return
list
(
set
(
p
[
"
project_code
"
]
for
p
in
self
.
products
.
values
()))
def
find_products
(
self
)
->
Iterator
[
SpooledProduct
]:
for
path
,
product
in
self
.
products
.
items
():
yield
ArchiveProduct
(
self
.
root
/
path
,
product
)
class
RestoreProductFinder
(
ProductFinder
):
"""
Used when there
'
s a single RestoreProduct that
'
s the entirety of `dir`,
whose metadata can be gleaned from the dir
'
s metadata.json.
Can easily be extended to the ALMA restore use-case;
should be extensible to multiple restores per delivery as well.
"""
def
__init__
(
self
,
dir
:
pathlib
.
Path
)
->
None
:
self
.
dir
=
dir
self
.
_metadata
=
self
.
parse_metadata
()
def
parse_metadata
(
self
)
->
ProductMetadata
:
metadata_file
=
json
.
loads
((
self
.
dir
/
"
metadata.json
"
).
read_bytes
())
project_metadata
=
metadata_file
[
"
projectMetadata
"
]
casa_log_path
=
list
(
self
.
dir
.
glob
(
"
*/working/casa-*.log
"
))
assert
len
(
casa_log_path
)
==
1
casa_log_name
=
casa_log_path
[
0
].
name
start_timestamp_match
=
re
.
match
(
r
"
^casa-(?P<timestamp>[0-9]+-[0-9]+).log$
"
,
casa_log_name
)
assert
start_timestamp_match
start_timestamp
=
pendulum
.
from_format
(
start_timestamp_match
.
group
(
"
timestamp
"
),
"
YYYYMMDD-HHmmss
"
)
start_mjd
=
date_to_mjd
(
start_timestamp
)
return
ProductMetadata
(
project_metadata
[
"
telescope
"
],
project_metadata
[
"
project_code
"
],
"
restored_cms
"
,
None
,
None
,
str
(
start_mjd
)
)
@property
def
projects
(
self
)
->
List
[
str
]:
return
[
self
.
_metadata
.
project
]
def
find_products
(
self
)
->
Iterator
[
SpooledProduct
]:
yield
RestoreProduct
(
self
.
dir
,
self
.
_metadata
)
def
date_to_mjd
(
timestamp
:
DateTime
)
->
float
:
return
0
This diff is collapsed.
Click to expand it.
apps/cli/executables/pexable/deliver/delivery/products.py
+
13
−
0
View file @
8e3ead95
...
...
@@ -197,6 +197,19 @@ class ArchiveProduct(SpooledProduct):
return
self
.
_metadata
class
RestoreProduct
(
SpooledProduct
):
def
__init__
(
self
,
path
:
pathlib
.
Path
,
metadata
:
ProductMetadataIF
):
super
().
__init__
(
path
)
self
.
_metadata
=
metadata
@property
def
metadata
(
self
)
->
ProductMetadataIF
:
return
self
.
_metadata
def
deliver_to
(
self
,
destination
:
Destination
):
return
super
().
deliver_to
(
destination
)
class
Calibration
(
SpooledProduct
):
"""
This class is intended for use when we do not have product metadata to use.
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment