diff --git a/buildout.cfg b/buildout.cfg index 67ba7fcb78876e772e16369101d7558c5ddf3097..dbd0f230c0c1d3158755df93c93a65d4ebe4ab85 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -1,6 +1,6 @@ # Main section # Accessible from the command line using 'buildout key=value' -# e.g. buildout parts=build_packages will specify build_packages +# e.g. buildout parts=build_pkgs will specify build_pkgs # as a part that should be installed [buildout] develop = recipes/setup_to_meta recipes/build_pkgs @@ -9,8 +9,8 @@ develop = recipes/setup_to_meta recipes/build_pkgs # Depends on gen_metadata # Depends on `name` in [buildout] to specify which package to install # Specify name via command line using -# `buildout parts=build_packages name={name}` -[build_packages] +# `buildout parts=build_pkgs name={name}` +[build_pkgs] => gen_metadata recipe = build_pkgs name = ${buildout:name} diff --git a/recipes/build_pkgs/build_pkgs.py b/recipes/build_pkgs/build_pkgs.py index bf116452b9e113b85734c164d1be0ce9402b0a13..984203d7d45d33f9e43f177fce565cbd533461ad 100644 --- a/recipes/build_pkgs/build_pkgs.py +++ b/recipes/build_pkgs/build_pkgs.py @@ -1,32 +1,17 @@ -import subprocess +import subprocess, os, glob def get_pkg_list(): """ Run a couple shell commands to parse the metadata directory for its packages. :return: List of packages in metadata directory """ - find_proc = subprocess.Popen(["find", "metadata", + find_proc = subprocess.run(["find", "metadata", "-name", "meta.yaml"], stdout=subprocess.PIPE) - return subprocess.check_output(["sed", "-e", "s:metadata/::", - "-e", "s:/meta.yaml::"], - stdin=find_proc.stdout).decode('utf-8').split('\n') + paths = find_proc.stdout.decode('utf-8') + fmt_paths = paths.replace("metadata/", "").replace("/meta.yaml", "") + return fmt_paths.split('\n') -def transfer_packages(): - """ - Use shell commands to transfer build archives to builder and update its conda package index. - """ - builder_addr = "builder.aoc.nrao.edu" - builder_path = "/home/builder.aoc.nrao.edu/content/conda/noarch" - subprocess.run(["ls", "pkgs/noarch/*.tar.bz2"]) - subprocess.run(["scp", "pkgs/noarch/*.tar.bz2", builder_addr + ':' + builder_path]) - cmd_cd = "cd {}".format(builder_path) - cmd_index = "conda index .." - cmd_chmod = "chmod -f 664 *" - subprocess.run(["ssh", builder_addr, - cmd_cd + " && " + - cmd_index + " && " + - cmd_chmod]) class Recipe: def __init__(self, buildout, name, options): @@ -57,7 +42,8 @@ class Recipe: subprocess.run(["conda", "build", "metadata/{}".format(p), "--output-folder", "pkgs/"], stdout=subprocess.PIPE) self.options.created("pkgs/") - transfer_packages() + + subprocess.run(["python3", "tools/transfer_to_builder.py"]) return self.options.created() diff --git a/recipes/setup_to_meta/setup_to_meta.py b/recipes/setup_to_meta/setup_to_meta.py index af65485abc6c82d28138fd34eeb0697685e9c59d..c011e624d9c0870926278199b7b9faad3aa709eb 100644 --- a/recipes/setup_to_meta/setup_to_meta.py +++ b/recipes/setup_to_meta/setup_to_meta.py @@ -147,7 +147,7 @@ def get_dirs(): :return: List of directories as strings. """ find = subprocess.run([ - 'find', '.', '-name', 'setup.py', '-not', '-path', './src/*' + 'find', '.', '-name', 'setup.py', '-not', '-path', './recipes/*' ], stdout=subprocess.PIPE) dirs = find.stdout.decode('utf-8').split('\n') dirs_cpy = dirs diff --git a/tools/transfer_to_builder.py b/tools/transfer_to_builder.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..085e81807b7f1d8343c5d470edc5405a8e4bf970 100644 --- a/tools/transfer_to_builder.py +++ b/tools/transfer_to_builder.py @@ -0,0 +1,46 @@ +import subprocess, paramiko, fnmatch, os +from scp import SCPClient + +def get_build_pkg_names(): + """ + Search through pkgs directory for built .tar.bz2 packages + :return: List of package archive file names + """ + pkg_names = [] + d = "pkgs/noarch/" + for file in os.listdir(d): + if fnmatch.fnmatch(file, "*.tar.bz2"): + pkg_names.append(d + file) + + return pkg_names + +def create_ssh_client(server): + client = paramiko.SSHClient() + client.load_system_host_keys() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect(server) + return client + +def transfer_packages(pkg_names): + """ + Use shell commands to transfer build archives to builder and update its conda package index. + """ + builder_addr = "builder.aoc.nrao.edu" + builder_path = "/home/builder.aoc.nrao.edu/content/conda/noarch" + ssh = create_ssh_client(builder_addr) + scp = SCPClient(ssh.get_transport()) + if len(pkg_names): + scp.put(' '.join(pkg_names), builder_path) + cmd_cd = "cd {}".format(builder_path) + cmd_index = "conda index .." + cmd_chmod = "chmod -f 664 *" + subprocess.run(["ssh", builder_addr, + cmd_cd + " && " + + cmd_index + " && " + + cmd_chmod]) + else: + print("No packages found in pkgs/noarch. Did conda build successfully build the package(s)?") + return + +if __name__ == "__main__": + transfer_packages(get_build_pkg_names()) \ No newline at end of file