From 70f1949c4be5f5a0f3260b8509ab008ade88a294 Mon Sep 17 00:00:00 2001
From: nhertz <nhertz@nrao.edu>
Date: Fri, 14 Aug 2020 16:07:40 -0600
Subject: [PATCH] Completed recipe for building packages and transferring them
to builder.
---
buildout.cfg | 6 ++--
recipes/build_pkgs/build_pkgs.py | 28 ++++------------
recipes/setup_to_meta/setup_to_meta.py | 2 +-
tools/transfer_to_builder.py | 46 ++++++++++++++++++++++++++
4 files changed, 57 insertions(+), 25 deletions(-)
diff --git a/buildout.cfg b/buildout.cfg
index 67ba7fcb7..dbd0f230c 100644
--- a/buildout.cfg
+++ b/buildout.cfg
@@ -1,6 +1,6 @@
# Main section
# Accessible from the command line using 'buildout key=value'
-# e.g. buildout parts=build_packages will specify build_packages
+# e.g. buildout parts=build_pkgs will specify build_pkgs
# as a part that should be installed
[buildout]
develop = recipes/setup_to_meta recipes/build_pkgs
@@ -9,8 +9,8 @@ develop = recipes/setup_to_meta recipes/build_pkgs
# Depends on gen_metadata
# Depends on `name` in [buildout] to specify which package to install
# Specify name via command line using
-# `buildout parts=build_packages name={name}`
-[build_packages]
+# `buildout parts=build_pkgs name={name}`
+[build_pkgs]
=> gen_metadata
recipe = build_pkgs
name = ${buildout:name}
diff --git a/recipes/build_pkgs/build_pkgs.py b/recipes/build_pkgs/build_pkgs.py
index bf116452b..984203d7d 100644
--- a/recipes/build_pkgs/build_pkgs.py
+++ b/recipes/build_pkgs/build_pkgs.py
@@ -1,32 +1,17 @@
-import subprocess
+import subprocess, os, glob
def get_pkg_list():
"""
Run a couple shell commands to parse the metadata directory for its packages.
:return: List of packages in metadata directory
"""
- find_proc = subprocess.Popen(["find", "metadata",
+ find_proc = subprocess.run(["find", "metadata",
"-name", "meta.yaml"],
stdout=subprocess.PIPE)
- return subprocess.check_output(["sed", "-e", "s:metadata/::",
- "-e", "s:/meta.yaml::"],
- stdin=find_proc.stdout).decode('utf-8').split('\n')
+ paths = find_proc.stdout.decode('utf-8')
+ fmt_paths = paths.replace("metadata/", "").replace("/meta.yaml", "")
+ return fmt_paths.split('\n')
-def transfer_packages():
- """
- Use shell commands to transfer build archives to builder and update its conda package index.
- """
- builder_addr = "builder.aoc.nrao.edu"
- builder_path = "/home/builder.aoc.nrao.edu/content/conda/noarch"
- subprocess.run(["ls", "pkgs/noarch/*.tar.bz2"])
- subprocess.run(["scp", "pkgs/noarch/*.tar.bz2", builder_addr + ':' + builder_path])
- cmd_cd = "cd {}".format(builder_path)
- cmd_index = "conda index .."
- cmd_chmod = "chmod -f 664 *"
- subprocess.run(["ssh", builder_addr,
- cmd_cd + " && " +
- cmd_index + " && " +
- cmd_chmod])
class Recipe:
def __init__(self, buildout, name, options):
@@ -57,7 +42,8 @@ class Recipe:
subprocess.run(["conda", "build", "metadata/{}".format(p), "--output-folder", "pkgs/"],
stdout=subprocess.PIPE)
self.options.created("pkgs/")
- transfer_packages()
+
+ subprocess.run(["python3", "tools/transfer_to_builder.py"])
return self.options.created()
diff --git a/recipes/setup_to_meta/setup_to_meta.py b/recipes/setup_to_meta/setup_to_meta.py
index af65485ab..c011e624d 100644
--- a/recipes/setup_to_meta/setup_to_meta.py
+++ b/recipes/setup_to_meta/setup_to_meta.py
@@ -147,7 +147,7 @@ def get_dirs():
:return: List of directories as strings.
"""
find = subprocess.run([
- 'find', '.', '-name', 'setup.py', '-not', '-path', './src/*'
+ 'find', '.', '-name', 'setup.py', '-not', '-path', './recipes/*'
], stdout=subprocess.PIPE)
dirs = find.stdout.decode('utf-8').split('\n')
dirs_cpy = dirs
diff --git a/tools/transfer_to_builder.py b/tools/transfer_to_builder.py
index e69de29bb..085e81807 100644
--- a/tools/transfer_to_builder.py
+++ b/tools/transfer_to_builder.py
@@ -0,0 +1,46 @@
+import subprocess, paramiko, fnmatch, os
+from scp import SCPClient
+
+def get_build_pkg_names():
+ """
+ Search through pkgs directory for built .tar.bz2 packages
+ :return: List of package archive file names
+ """
+ pkg_names = []
+ d = "pkgs/noarch/"
+ for file in os.listdir(d):
+ if fnmatch.fnmatch(file, "*.tar.bz2"):
+ pkg_names.append(d + file)
+
+ return pkg_names
+
+def create_ssh_client(server):
+ client = paramiko.SSHClient()
+ client.load_system_host_keys()
+ client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ client.connect(server)
+ return client
+
+def transfer_packages(pkg_names):
+ """
+ Use shell commands to transfer build archives to builder and update its conda package index.
+ """
+ builder_addr = "builder.aoc.nrao.edu"
+ builder_path = "/home/builder.aoc.nrao.edu/content/conda/noarch"
+ ssh = create_ssh_client(builder_addr)
+ scp = SCPClient(ssh.get_transport())
+ if len(pkg_names):
+ scp.put(' '.join(pkg_names), builder_path)
+ cmd_cd = "cd {}".format(builder_path)
+ cmd_index = "conda index .."
+ cmd_chmod = "chmod -f 664 *"
+ subprocess.run(["ssh", builder_addr,
+ cmd_cd + " && " +
+ cmd_index + " && " +
+ cmd_chmod])
+ else:
+ print("No packages found in pkgs/noarch. Did conda build successfully build the package(s)?")
+ return
+
+if __name__ == "__main__":
+ transfer_packages(get_build_pkg_names())
\ No newline at end of file
--
GitLab