#! /usr/bin/env python """bundle delivery operations. Provide operations related to the delivery of bundles, like generating diff files between bundle versions, or publishing a bundle and its associated files. """ import datetime import git import os import re import shutil import subprocess import sys import tarfile import tempfile # The path to the bundle storage area. BUNDLE_STORAGE_PATH = "//GEVREY/LIVRAISONS" # In the bundle storage, the directory where SPC300 bundles are stored. BUNDLE_STORAGE_SPC300_BUNDLES_DIR = "SPC300/SPiDBundle" # In the bundle storage, the format of the name of a each directory containing # a bundle. BUNDLE_STORAGE_BUNDLE_DIR_FMT = "{date}_SPiDBundle_{tagname}" # In the bundle storage, the format of the filename of a bundle. BUNDLE_STORAGE_BUNDLE_FILE_FMT = "SPiDBundle-spc300-{tagname}.tar.bz2" # The format of the filename of the resulting diff between two bundles. DIFF_FILE_FMT = "SPiDBundle-spc300-{tagname_from}-{tagname_to}.diff" # How to reach the git root directory from where this file is. GIT_ROOT_RELATIVE_PATH = ".." class BundleStorage: """The storage area for bundles.""" def __init__(self, dir_): """Instantiate. dir_: the directory where the SPC300 bundles are stored. """ self._dir = dir_ def _create_bundle_dir(self, tag): """Create the bundle directory for a tag.""" bundle_dir = BUNDLE_STORAGE_BUNDLE_DIR_FMT.format( date=datetime.date.today().strftime("%Y_%m_%d"), tagname=tag.name) bundle_dir = os.path.join(self._dir, bundle_dir) print "Creating \"{0}\"...".format(bundle_dir) os.mkdir(bundle_dir) return bundle_dir @classmethod def _find_in_mounted_filesystems(cls): """Find the bundle storage by looking into mounted filesystems.""" dir_ = None storage_re = re.compile( "{0}/?\s+(.*)\s+(cifs|smbfs)\s+.*$".format(BUNDLE_STORAGE_PATH), re.IGNORECASE) proc_mount = open("/proc/mounts") if proc_mount: for line in proc_mount.readlines(): m = storage_re.match(line) if m: dir_ = m.group(1) break proc_mount.close() return dir_ def _get_bundle_dir(self, tag): """Get the bundle directory for a tag.""" b_re = re.compile( BUNDLE_STORAGE_BUNDLE_DIR_FMT.format(date=".*", tagname=tag.name)) bundle_dirs = [] for d in os.listdir(self._dir): if b_re.match(d): bundle_dirs.append(os.path.join(self._dir, d)) # Don't break yet. Continue, to see if we find other # directories (which we shouldn't!) nb = len(bundle_dirs) if nb > 1: raise RuntimeError( "Unexpected result: more than one bundle directory found for " "tag \"{0}\"".format(tag.name)) elif nb == 0: return None else: return bundle_dirs[0] @classmethod def find(cls): """Find the bundle storage area. Find the bundle storage area either by looking into mount points, or by asking the user. """ dir_ = cls._find_in_mounted_filesystems() if dir_ is None: dir_ = raw_input("Path to bundle storage?\n--> ") dir_ = os.path.join(dir_, BUNDLE_STORAGE_SPC300_BUNDLES_DIR) if not os.path.exists(dir_): raise RuntimeError( "Cannot find SPC300 bundle storage: \"{0}\"".format(dir_)) return cls(dir_) def get_bundle(self, tag, dest_dir): """Download a bundle file from the bundle storage. tag: tag of the bundle to download. dest_dir: directory where to put the downloaded bundle. Return the path to the downloaded bundle. """ bundle_file = None bundle_dir = self._get_bundle_dir(tag) if bundle_dir: bundle_file = os.path.join( bundle_dir, BUNDLE_STORAGE_BUNDLE_FILE_FMT.format(tagname=tag.name)) if (bundle_file is None) or (not os.path.exists(bundle_file)): raise RuntimeError( "Bundle not found for tag \"{0}\"".format(tag.name)) dst = os.path.join(dest_dir, os.path.basename(bundle_file)) print "Copying \"{0}\"...".format(bundle_file) shutil.copyfile(bundle_file, dst) return dst def put_bundle_files(self, bundle, diff, *other): """Upload a bundle + diff + other files to the bundle storage. bundle: path to the bundle file to upload. diff: path to the diff file to upload. other: paths to other files to upload. The tag is determined from the filename of the bundle. """ def _check_extension(file, desc, extension): """Check that a file has the expected extension.""" root, ext = os.path.splitext(file) if not (ext == extension): raise RuntimeError("{0} file doesn't have expected \"{1}\" " "extension".format(desc, extension)) _check_extension(bundle, "bundle", ".bz2") _check_extension(diff, "diff", ".diff") tag = Tag.from_filename(bundle) if tag is None: raise RuntimeError("Cannot determine tag from bundle filename.") bundle_dir = self._get_bundle_dir(tag) if bundle_dir is None: bundle_dir = self._create_bundle_dir(tag) for f in [bundle, diff] + list(other): dst = os.path.join(bundle_dir, os.path.basename(f)) if os.path.exists(dst): ans = raw_input("Overwrite \"{0}\" [y/n]? ".format(dst)) if ans != "y": continue print "Copying \"{0}\"...".format(f) shutil.copyfile(f, dst) class Tag: """The tag of a bundle. Attributes: name: the name of a tag. e.g. "av-1.2.0". project: the project of a tag. e.g. "av". version: the version of a tag. e.g. (1, 2, 0) or (1, 1, 3, "tmp"). """ def __init__(self, name, project, version): self.name = name self.project = project self.version = version @classmethod def from_tagname(cls, name): """Create a tag using a tagname from the source repository.""" m = re.match(r"(av)-(\d+\.\d+\.\d+)-?(.*)", name) if not m: return None project = m.group(1) version = [ int(i) for i in m.group(2).split(".") ] if m.group(3): version.append(m.group(3)) version = tuple(version) return cls(name, project, version) @classmethod def from_filename(cls, filename): """Create a tag using a bundle filename.""" pattern = (".*" + BUNDLE_STORAGE_BUNDLE_FILE_FMT.format(tagname="(.*)") + "$") m = re.match(pattern, filename) if not m: return None tag_name = m.group(1) return cls.from_tagname(tag_name) def is_main_version(self): """Is it the tag of a main version? e.g. av-1.1.2 -> True av-1.1.3-tmp -> False eoc-0.6.7 -> True eoc-0.6.7-t2746 -> False """ return len(self.version) <= 3 @staticmethod def compare_version(tag1, tag2): """Compare the version of two tags of the same project. Return: -1, if tag1's version is less than tag2's version. 0, if tag1 and tag2 have the same version. +1, if tag1's version is greater than tag2's version. """ if tag1.project != tag2.project: raise RuntimeError("Cannot compare tags from different projects") return (tag1.version > tag2.version) - (tag1.version < tag2.version) @staticmethod def get_preceding_tag(git_repo, tag): """Get the tag preceding a given tag. git_repo: the git repo to use to find the preceding tag. tag: the tag for which the preceding tag is searched. Return the preceding tag. e.g. the preceding tag of av-1.1.0 is av-1.0.14. The preceding tag is of the same project as the input tag. preceding_tag.is_main_version() == True. """ pre_tag = None # The preceding tag. for repo_tag in git_repo.tags: t = Tag.from_tagname(repo_tag.name) if ((t is None) or (t.project != tag.project) or (not t.is_main_version())): continue if ((Tag.compare_version(t, tag) < 0) and ((pre_tag is None) or (Tag.compare_version(t, pre_tag) > 0))): pre_tag = t return pre_tag class BundleDiff: """Difference between bundles.""" def __init__(self, git_repo, bundle_storage): """Instantiate. git_repo: a git repository (Git.Repo). bundle_storage: a BundleStorage. """ self._work_dir = None self._git_repo = git_repo self._bundle_storage = bundle_storage def _create_work_dir(self): """Create a directory where to do internal work.""" self._work_dir = tempfile.mkdtemp(prefix='bundle-delivery-diff-') self._work_sub_dir1 = os.path.join(self._work_dir, "a") os.mkdir(self._work_sub_dir1) self._work_sub_dir2 = os.path.join(self._work_dir, "b") os.mkdir(self._work_sub_dir2) def _clean_work_dir(self): """Clean the directory where internal work is done.""" shutil.rmtree(self._work_dir) def _diff(self, dir1, dir2, output_file, strip_common_prefix=False): """Generate a diff file between two directories. dir1: first directory for the diff. dir2: second directory for the diff. output_file: path of the output diff file. strip_common_prefix: If True, the common prefix between paths dir1 and dir2 will be stripped from the diff output (e.g. for "/dir/a/dir1" and "/dir/b/dir2", the diff output will contain "a/dir1" and "b/dir2). """ # Check that directories exist, because the diff command won't # complain, due to the -N option. def _check_dir(dir_): if not os.path.exists(dir_): raise RuntimeError("Directory \"{0}\" not found.".format(dir_)) _check_dir(dir1) _check_dir(dir2) print "Creating diff file \"{0}\"...".format(output_file) diff_log = os.path.join(self._work_dir, "diff.log") diff_output = os.path.join(self._work_dir, "diff_output") command = "" if strip_common_prefix: common_prefix = os.path.commonprefix([dir1, dir2]) dir1 = os.path.relpath(dir1, start=common_prefix) dir2 = os.path.relpath(dir2, start=common_prefix) command = "cd {0};".format(common_prefix) command += "diff -aurN {0} {1} > {2}".format(dir1, dir2, diff_output) fd = open(diff_log, "w") ret = subprocess.call(command, stdout = fd, stderr = fd, shell = True) fd.close() shutil.move(diff_output, output_file) def _extract_archive(self, file, dest_dir): """Uncompress an archive compressed with bzip2. file: the archive. dest_dir: the directory where to put the decompressed contents of the archive. The function expects that the archive decompresses into one directory. """ print "Decompressing \"{0}\"...".format(os.path.basename(file)) extract_dir = os.path.join(dest_dir, "extract_tmp") tar = tarfile.open(file, "r:bz2") tar.extractall(extract_dir) tar.close() dirs = os.listdir(extract_dir) if len(dirs) != 1: raise RuntimeError("Archive isn't contained in one directory, " "as was expected.") tmp = os.path.join(extract_dir, dirs[0]) final = os.path.join(dest_dir, dirs[0]) shutil.move(tmp, final) return final def _id_tag_or_file(self, bundle): """Identify whether what was provided is a tag name or a filename.""" file = None # First, let's suppose it's a tag. tag = Tag.from_tagname(bundle) if tag is None: # Maybe the filename of a bundle, then? tag = Tag.from_filename(bundle) if tag: file = bundle return (tag, file) def gen_diff_file(self, bundle1, bundle2): """Generate a diff file between two bundles. bundle1: The first bundle. If None, bundle1 will be set to the bundle preceding bundle2. bundle2: The second bundle. """ if bundle2 is None: raise RuntimeError("bundle2 cannot be \"None\"") (tag2, file2) = self._id_tag_or_file(bundle2) if tag2 is None: raise RuntimeError( "Cannot determine if \"{0}\" is a tag or a file." .format(bundle2)) if bundle1: (tag1, file1) = self._id_tag_or_file(bundle1) if tag1 is None: raise RuntimeError( "Cannot determine if \"{0}\" is a tag or a file." .format(bundle1)) else: print "Finding preceding tag of \"{0}\"... ".format(tag2.name) tag1 = Tag.get_preceding_tag(self._git_repo, tag2) if tag1: print "Preceding tag: \"{0}\"".format(tag1.name) else: raise RuntimeError("Cannot find preceding tag.") file1 = None self._create_work_dir() if not file1: file1 = self._bundle_storage.get_bundle(tag1, self._work_sub_dir1) if not file2: file2 = self._bundle_storage.get_bundle(tag2, self._work_sub_dir2) dir1 = self._extract_archive(file1, self._work_sub_dir1) # Delete file1 if we got it from bundle storage if file1 != bundle1: os.remove(file1) dir2 = self._extract_archive(file2, self._work_sub_dir2) # Delete file2 if we got it from bundle storage if file2 != bundle2: os.remove(file2) diff_file = DIFF_FILE_FMT.format(tagname_from=tag1.name, tagname_to=tag2.name) self._diff(os.path.join(dir1, "bundle"), os.path.join(dir2, "bundle"), diff_file, strip_common_prefix=True) self._clean_work_dir() return diff_file class BundleDeliveryCmd: """BundleDelivery commands.""" def __init__(self): from optparse import OptionParser usage = r"""%prog diff|push args... %prog diff bundle1-file|bundle1-tag bundle2-file|bundle2-tag Generate a diff file between two bundles. For each bundle, either a tag or the file can be provided. %prog diff bundle-file|bundle-tag Generate a diff file between the specified bundle and the bundle preceding it. %prog push bundle-file diff-file [other-file1 ... other-fileN] Publish the bundle and its associated files (diff file + eventually other files) to the bundle storage area. """ self._commands = dict((n[4:], getattr(self, n)) for n in dir(self) if n.startswith('run_')) self._parser = OptionParser(usage=usage) def run(self, args): if not args: self._parser.error("Too few arguments") options, args = self._parser.parse_args(args) cmd = args[0] if cmd not in self._commands: self._parser.error("Unknown command \"{0}\".".format(cmd)) self._commands[cmd](args[1:]) def run_diff(self, args): """Run the diff procedure.""" if len(args) not in [1, 2]: self._parser.error("Invalid number of arguments.") bundle_storage = BundleStorage.find() repository = os.path.join(os.path.abspath(__file__), GIT_ROOT_RELATIVE_PATH) git_repo = git.Repo(repository) b = BundleDiff(git_repo, bundle_storage) if len(args) == 2: b.gen_diff_file(args[0], args[1]) else: b.gen_diff_file(None, args[0]) def run_push(self, args): """Run the push procedure.""" if len(args) < 2: self._parser.error("Invalid number of arguments.") bundle_storage = BundleStorage.find() bundle_storage.put_bundle_files(*args) if __name__ == "__main__": try: BundleDeliveryCmd().run(sys.argv[1:]) except RuntimeError, e: print >> sys.stderr, "Error:", e sys.exit(1)