WS-387 and WS-541: Production Pyramid configuration

docker-compose.local.yml

@@ -43,6 +43,7 @@ services:
       - ./delivery_root:/tmp/delivery_root
       - ./lustre/aoc/cluster/pipeline/docker/workspaces:/lustre/aoc/cluster/pipeline/docker/workspaces
 
+  # To enable NGAS container, run: `docker compose -f docker-compose.local.yml --profile ngas up`
   ngas:
     build:
       context: .
@@ -53,6 +54,8 @@ services:
       - "7777:7777"
     volumes:
       - ./local_ngas_root:/home/ngas/NGAS
+    profiles:
+      - ngas
 
   schema:
     build:
@@ -65,17 +68,19 @@ services:
       - ./docker.properties:/home/ssa/capo/docker.properties
 
   # Used for doing a DB migration on dsoc-dev
-  # Uncomment this and run the command `docker compose -f docker-compose.local.yml up schema-dev` and the migration will run
-#  schema-dev:
-#    build:
-#      context: ./schema/
-#      dockerfile: Dockerfile.local
-#    command: ["./bin/run-migrations.sh", "dsoc-dev"]
-#    depends_on:
-#      - db
-#    volumes:
-#      - ./schema:/code/schema
-#      - ~/.capo:/home/casa/capo
+  # Run the command `docker compose -f docker-compose.local.yml --profile schema-dev up schema-dev` and the migration will run
+  schema-dev:
+    build:
+      context: ./schema/
+      dockerfile: Dockerfile.local
+    command: ["./bin/run-migrations.sh", "dsoc-dev"]
+    depends_on:
+      - db
+    volumes:
+      - ./schema:/code/schema
+      - ~/.capo:/home/casa/capo
+    profiles:
+      - schema-dev
 
   workflow:
     build:

services/capability/Dockerfile

@@ -2,6 +2,10 @@
 ARG TAGNAME="tmp"
 FROM cache:${TAGNAME}
 
+# Build arg that sets environment; sets to "dev" if no build arg is given
+ARG env=dev
+ENV ENV=${env}
+
 # Change working directory to /code
 WORKDIR /code
 
@@ -24,4 +28,4 @@ RUN python setup.py develop --user
 # Gets reset to proper environment's profile in the deploy stage
 ENV CAPO_PROFILE docker
 
-CMD ["pserve", "--reload", "development.ini"]
+CMD ["pserve", "--reload", "${ENV}.ini"]

services/capability/Dockerfile.local

@@ -14,4 +14,4 @@ RUN python setup.py develop --user
 ARG capo_env=docker
 ENV CAPO_PROFILE $capo_env
 
-CMD ["pserve", "--reload", "development.ini"]
+CMD ["pserve", "--reload", "dev.ini"]

services/capability/README.md

@@ -8,10 +8,9 @@ Then run:
     
 Then you should be able to run this with:
 
-    env CAPO_PROFILE=local pserve --reload development.ini
+    env CAPO_PROFILE=local pserve --reload dev.ini
 
 Once there, try the following URLs:
 
 - http://0.0.0.0:3456/capabilities
 - http://0.0.0.0:3456/capability/request?capability=null
-

services/capability/bin/boot-local.sh

-#! /bin/bash
-
-# Set failfast
-set -e
-set -o pipefail
-
-# FOR USE WITH DOCKER DEVELOPMENT ENVIRONMENT ONLY.
-# Capability Service
-
-# Python library installation
-pip install -r requirements.txt 
-python setup.py develop
-
-# Start development server
-pserve --reload development.ini

services/capability/production.ini → services/capability/prod.ini

@@ -19,4 +19,41 @@ pyramid.includes = pyramid_tm
 
 [server:main]
 use = egg:waitress#main
-listen = 0.0.0.0:6543
+listen = 0.0.0.0:3457
+
+###
+# logging configuration
+# https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html
+###
+
+[loggers]
+keys = root, capability, sqlalchemy
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+
+[logger_capability]
+level = WARN
+handlers =
+qualname = capability
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s

services/notification/Dockerfile

@@ -2,6 +2,10 @@
 ARG TAGNAME="tmp"
 FROM cache:${TAGNAME}
 
+# Build arg that sets environment; sets to "dev" if no build arg is given
+ARG env=dev
+ENV ENV=${env}
+
 # Change working directory to /code
 WORKDIR /code
 
@@ -24,4 +28,4 @@ RUN python setup.py develop --user
 # Gets reset to proper environment's profile in the deploy stage
 ENV CAPO_PROFILE docker
 
-CMD ["pserve", "--reload", "development.ini"]
+CMD ["pserve", "--reload", "${ENV}.ini"]

services/notification/Dockerfile.local

@@ -14,4 +14,4 @@ RUN python setup.py develop --user
 ARG capo_env=docker
 ENV CAPO_PROFILE $capo_env
 
-CMD ["pserve", "--reload", "development.ini"]
+CMD ["pserve", "--reload", "dev.ini"]

services/notification/bin/boot-local.sh

-#! /bin/bash
-
-# Set failfast
-set -e
-set -o pipefail
-
-# FOR USE WITH DOCKER DEVELOPMENT ENVIRONMENT ONLY.
-# Workflow Service
-
-# Python library installation
-pip install -r requirements.txt 
-python setup.py develop
-
-# Start development server
-pserve --reload development.ini

services/notification/prod.ini

+###
+# app configuration
+# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html
+###
+
+[app:main]
+use = egg:ssa-notification
+
+session.cookie_expires = true
+session.auto = true
+
+pyramid.reload_templates = false
+pyramid.debug_authorization = false
+pyramid.debug_notfound = false
+pyramid.debug_routematch = false
+pyramid.default_locale_name = en
+pyramid.includes = pyramid_tm
+
+
+[server:main]
+use = egg:waitress#main
+listen = 0.0.0.0:3458
+
+
+###
+# logging configuration
+# https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html
+###
+
+[loggers]
+keys = root, notifcation, sqlalchemy
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+
+[logger_notification]
+level = WARN
+handlers =
+qualname = notification
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s

services/workflow/Dockerfile.oracle.local

@@ -42,4 +42,4 @@ WORKDIR /packages/apps/cli/executables/productfetcher/tests
 RUN pytest
 # RUN python setup.py develop
 
-CMD ["pserve", "--reload", "/code/development.ini"]
+CMD ["pserve", "--reload", "/code/dev.ini"]

services/workflow/README.md

@@ -7,11 +7,10 @@ Then, back in services/workflow, run:
     `python setup.py develop`
     
 Then you should be able to run this with:
-    `env CAPO_PROFILE=local pserve --reload development.ini`
+    `env CAPO_PROFILE=local pserve --reload dev.ini`
 
 Once there, try the following URLs:
 
 - http://0.0.0.0:3456/workflows/requests/94
 - http://0.0.0.0:3456/workflows
 - http://0.0.0.0:3456/workflows/0
-

services/workflow/bin/boot-workflow.sh

@@ -26,5 +26,9 @@ done
 # Return to base /code directory
 cd /code
 
-# Start pserve normally
-pserve --reload development.ini
+# Start Pyramid server
+if [ "${CAPO_PROFILE}" = "dsoc-prod" ]; then
+    pserve --reload prod.ini
+else
+    pserve --reload dev.ini
+fi

services/workflow/production.ini → services/workflow/prod.ini

@@ -16,7 +16,45 @@ pyramid.debug_routematch = false
 pyramid.default_locale_name = en
 pyramid.includes = pyramid_tm
 
+retry.attempts = 3
 
 [server:main]
 use = egg:waitress#main
-listen = 0.0.0.0:6543
+listen = 0.0.0.0:3456
+
+###
+# logging configuration
+# https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html
+###
+
+[loggers]
+keys = root, workflow, sqlalchemy
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+
+[logger_workflow]
+level = WARN
+handlers =
+qualname = workflow
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s