Fix type of file content to be bytes, not strings. Move database connection into schema.

schema/versions/44d5bbbf2615_workspaces_init.py

@@ -21,7 +21,7 @@ def upgrade():
 
     op.create_table('workflow_templates',
                     sa.Column('filename', sa.String, primary_key=True, comment='the filename of the template'),
-                    sa.Column('content', sa.String, nullable=False, comment='the content of the template'),
+                    sa.Column('content', sa.LargeBinary, nullable=False, comment='the content of the template'),
                     sa.Column('workflow_name', sa.String, sa.ForeignKey('workflows.workflow_name'), primary_key=True,
                               comment='the name of the workflow this template belongs to'),
                     comment='Templates associated with workflows')
@@ -37,7 +37,7 @@ def upgrade():
                     sa.Column('workflow_request_id', sa.Integer, sa.ForeignKey('workflow_requests.workflow_request_id'),
                               comment='the id of the workflow request.'),
                     sa.Column('filename', sa.String, comment='the name of this file', nullable=False),
-                    sa.Column('content', sa.String, comment='the contents of the file', nullable=False),
+                    sa.Column('content', sa.LargeBinary, comment='the contents of the file', nullable=False),
                     comment='A man-to-many mapping table tracking which files were used for workflow requests.')
 
     op.create_unique_constraint('workflow_request_filenames_uniq', 'workflow_request_files',
@@ -45,7 +45,7 @@ def upgrade():
 
     # populate with some initial data
     op.execute("INSERT INTO workflows (workflow_name) VALUES ('null')")
-    op.execute("INSERT INTO workflow_templates (workflow_name, filename, content) VALUES ('null', 'null.sh', 'null {{arguments}}')")
+    op.execute("INSERT INTO workflow_templates (workflow_name, filename, content) VALUES ('null', 'null.sh', E'null {{arguments}}')")
 
 
 def downgrade():

services/workflow/src/workflow/server.py

@@ -6,7 +6,7 @@ from pyramid_beaker import session_factory_from_settings
 from pyramid.renderers import JSONP
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
-from workspaces.services import WorkflowInfo, WorkflowService
+from workspaces.services import WorkflowInfo, WorkflowService, get_session_factory, get_engine
 
 """
 Work done:
@@ -154,27 +154,6 @@ class WorkflowFilesRestService:
 # ---------------------------------------------------------
 
 
-def get_engine():
-    """
-    Generate the SQL Alchemy engine for us, using Capo.
-    :return:
-    """
-    capo = CapoConfig().settings('metadataDatabase')
-    url = capo.jdbcUrl.replace('jdbc:', '').replace('://', f'://{capo.jdbcUsername}:{capo.jdbcPassword}@')
-    return create_engine(url)
-
-
-def get_session_factory(engine):
-    """
-    Generate the SQL Alchemy session factory for us, using the supplied engine
-    :param engine:
-    :return:
-    """
-    factory = sessionmaker()
-    factory.configure(bind=engine)
-    return factory
-
-
 def get_tm_session(session_factory, transaction_manager):
     """
     Enable Zope's transaction manager on our session

shared/workspaces/src/workspaces/schema.py

@@ -7,7 +7,9 @@ from pathlib import Path
 from typing import Dict, List
 
 import sqlalchemy as sa
-from sqlalchemy.orm import relationship
+from pycapo import CapoConfig
+from sqlalchemy import create_engine
+from sqlalchemy.orm import relationship, sessionmaker
 from sqlalchemy.ext.declarative import declarative_base
 
 
@@ -16,15 +18,22 @@ class AbstractFile:
     Abstract file is exactly that, an abstract concept of what a file is, to be
     returned from various non-filesystem places.
     """
-    def __init__(self, filename, content):
+    filename: str
+    content: bytes
+
+    def __init__(self, filename: str, content: bytes):
         self.filename, self.content = filename, content
 
     def write_to(self, directory: Path):
-        raise NotImplementedError('Actually save the file into the directory passed')
+        (directory / self.filename).write_bytes(self.content)
 
     def __json__(self, request):
         return {'filename': self.filename, 'content': self.content}
 
+    @classmethod
+    def from_path(cls, path: Path) -> 'AbstractFile':
+        return cls(path.name, path.read_bytes())
+
 
 Base = declarative_base()
 
@@ -42,6 +51,9 @@ class Workflow(Base):
     def __json__(self, request):
         return {'workflow_name': self.workflow_name, 'templates': self.templates}
 
+    def __repr__(self):
+        return f'<Workflow workflow_name={self.workflow_name}>'
+
     def render_templates(self, argument: Dict, files: List[Path]) -> List[AbstractFile]:
         """
         Render the templates associated with this workflow
@@ -49,7 +61,7 @@ class Workflow(Base):
         :param files:    the files to be processed
         :return:         a list of rendered templates
         """
-        raise NotImplementedError(f'{self.__class__.__name__}.{inspect.stack()[0][3]}')
+        raise NotImplementedError
 
 
 class WorkflowTemplate(Base):
@@ -58,7 +70,7 @@ class WorkflowTemplate(Base):
     """
     __tablename__ = 'workflow_templates'
     filename = sa.Column('filename', sa.String, primary_key=True)
-    content = sa.Column('content', sa.String, nullable=False)
+    content = sa.Column('content', sa.LargeBinary, nullable=False)
     workflow_name = sa.Column('workflow_name', sa.String, sa.ForeignKey('workflows.workflow_name'), primary_key=True)
 
     def render(self, argument: Dict) -> AbstractFile:
@@ -68,10 +80,17 @@ class WorkflowTemplate(Base):
     def template_file(self) -> AbstractFile:
         return AbstractFile(self.filename, self.content)
 
+    @template_file.setter
+    def template_file(self, file: AbstractFile):
+        self.filename, self.content = file.filename, file.content
+
     def __json__(self, request):
         # Defer to the actual template file's contents for JSON conversion
         return self.template_file.__json__(request)
 
+    def __repr__(self):
+        return f'<WorkflowTemplate filename={self.filename} for workflow={self.workflow_name}>'
+
 
 class WorkflowRequest(Base):
     """
@@ -89,11 +108,40 @@ class WorkflowRequestFile(Base):
     A Workflow Request File is a file supplied by the user and attached to the request they have submitted.
     """
     __tablename__ = 'workflow_request_files'
-    workflow_request_id = sa.Column('workflow_request_id', sa.Integer, sa.ForeignKey('workflow_requests.workflow_request_id'),
+    workflow_request_id = sa.Column('workflow_request_id', sa.Integer,
+                                    sa.ForeignKey('workflow_requests.workflow_request_id'),
                                     primary_key=True)
     filename = sa.Column('filename', sa.String, primary_key=True)
-    content = sa.Column('content', sa.String, nullable=False)
+    content = sa.Column('content', sa.LargeBinary, nullable=False)
 
     @property
     def file(self) -> AbstractFile:
         return AbstractFile(self.filename, self.content)
+
+    @file.setter
+    def set_file(self, file: AbstractFile):
+        self.filename, self.content = file.filename, file.content
+
+    def __repr__(self):
+        return f'<WorkflowRequestFile filename={self.filename}>'
+
+
+def get_engine():
+    """
+    Generate the SQL Alchemy engine for us, using Capo.
+    :return:
+    """
+    capo = CapoConfig().settings('metadataDatabase')
+    url = capo.jdbcUrl.replace('jdbc:', '').replace('://', f'://{capo.jdbcUsername}:{capo.jdbcPassword}@')
+    return create_engine(url)
+
+
+def get_session_factory(engine):
+    """
+    Generate the SQL Alchemy session factory for us, using the supplied engine
+    :param engine:
+    :return:
+    """
+    factory = sessionmaker()
+    factory.configure(bind=engine)
+    return factory