Completed recipe for building packages and transferring them to builder.

buildout.cfg

 # Main section
 # Accessible from the command line using 'buildout key=value'
-# e.g. buildout parts=build_packages will specify build_packages
+# e.g. buildout parts=build_pkgs will specify build_pkgs
 # as a part that should be installed
 [buildout]
 develop = recipes/setup_to_meta recipes/build_pkgs
@@ -9,8 +9,8 @@ develop = recipes/setup_to_meta recipes/build_pkgs
 # Depends on gen_metadata
 # Depends on `name` in [buildout] to specify which package to install
 # Specify name via command line using
-# `buildout parts=build_packages name={name}`
-[build_packages]
+# `buildout parts=build_pkgs name={name}`
+[build_pkgs]
 => gen_metadata
 recipe = build_pkgs
 name = ${buildout:name}

recipes/build_pkgs/build_pkgs.py

-import subprocess
+import subprocess, os, glob
 
 def get_pkg_list():
     """
     Run a couple shell commands to parse the metadata directory for its packages.
     :return: List of packages in metadata directory
     """
-    find_proc = subprocess.Popen(["find", "metadata",
+    find_proc = subprocess.run(["find", "metadata",
                                   "-name", "meta.yaml"],
                                  stdout=subprocess.PIPE)
-    return subprocess.check_output(["sed", "-e", "s:metadata/::",
-                                    "-e", "s:/meta.yaml::"],
-                                   stdin=find_proc.stdout).decode('utf-8').split('\n')
+    paths = find_proc.stdout.decode('utf-8')
+    fmt_paths = paths.replace("metadata/", "").replace("/meta.yaml", "")
+    return fmt_paths.split('\n')
 
-def transfer_packages():
-    """
-    Use shell commands to transfer build archives to builder and update its conda package index.
-    """
-    builder_addr = "builder.aoc.nrao.edu"
-    builder_path = "/home/builder.aoc.nrao.edu/content/conda/noarch"
-    subprocess.run(["ls", "pkgs/noarch/*.tar.bz2"])
-    subprocess.run(["scp", "pkgs/noarch/*.tar.bz2", builder_addr + ':' + builder_path])
-    cmd_cd = "cd {}".format(builder_path)
-    cmd_index = "conda index .."
-    cmd_chmod = "chmod -f 664 *"
-    subprocess.run(["ssh", builder_addr,
-                    cmd_cd + " && " +
-                    cmd_index + " && " +
-                    cmd_chmod])
 
 class Recipe:
     def __init__(self, buildout, name, options):
@@ -57,7 +42,8 @@ class Recipe:
             subprocess.run(["conda", "build", "metadata/{}".format(p), "--output-folder", "pkgs/"],
                            stdout=subprocess.PIPE)
         self.options.created("pkgs/")
-        transfer_packages()
+
+        subprocess.run(["python3", "tools/transfer_to_builder.py"])
 
         return self.options.created()
 

recipes/setup_to_meta/setup_to_meta.py

@@ -147,7 +147,7 @@ def get_dirs():
     :return: List of directories as strings.
     """
     find = subprocess.run([
-        'find', '.', '-name', 'setup.py', '-not', '-path', './src/*'
+        'find', '.', '-name', 'setup.py', '-not', '-path', './recipes/*'
     ], stdout=subprocess.PIPE)
     dirs = find.stdout.decode('utf-8').split('\n')
     dirs_cpy = dirs

tools/transfer_to_builder.py

+import subprocess, paramiko, fnmatch, os
+from scp import SCPClient
+
+def get_build_pkg_names():
+    """
+    Search through pkgs directory for built .tar.bz2 packages
+    :return: List of package archive file names
+    """
+    pkg_names = []
+    d = "pkgs/noarch/"
+    for file in os.listdir(d):
+        if fnmatch.fnmatch(file, "*.tar.bz2"):
+            pkg_names.append(d + file)
+
+    return pkg_names
+
+def create_ssh_client(server):
+    client = paramiko.SSHClient()
+    client.load_system_host_keys()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    client.connect(server)
+    return client
+
+def transfer_packages(pkg_names):
+    """
+    Use shell commands to transfer build archives to builder and update its conda package index.
+    """
+    builder_addr = "builder.aoc.nrao.edu"
+    builder_path = "/home/builder.aoc.nrao.edu/content/conda/noarch"
+    ssh = create_ssh_client(builder_addr)
+    scp = SCPClient(ssh.get_transport())
+    if len(pkg_names):
+        scp.put(' '.join(pkg_names), builder_path)
+        cmd_cd = "cd {}".format(builder_path)
+        cmd_index = "conda index .."
+        cmd_chmod = "chmod -f 664 *"
+        subprocess.run(["ssh", builder_addr,
+                        cmd_cd + " && " +
+                        cmd_index + " && " +
+                        cmd_chmod])
+    else:
+        print("No packages found in pkgs/noarch. Did conda build successfully build the package(s)?")
+        return
+
+if __name__ == "__main__":
+    transfer_packages(get_build_pkg_names())
\ No newline at end of file