Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'mk/maint-parse-careful' (ee4f06c)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19def die(msg):
  20    if verbose:
  21        raise Exception(msg)
  22    else:
  23        sys.stderr.write(msg + "\n")
  24        sys.exit(1)
  25
  26def write_pipe(c, str):
  27    if verbose:
  28        sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30    pipe = os.popen(c, 'w')
  31    val = pipe.write(str)
  32    if pipe.close():
  33        die('Command failed: %s' % c)
  34
  35    return val
  36
  37def read_pipe(c, ignore_error=False):
  38    if verbose:
  39        sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41    pipe = os.popen(c, 'rb')
  42    val = pipe.read()
  43    if pipe.close() and not ignore_error:
  44        die('Command failed: %s' % c)
  45
  46    return val
  47
  48
  49def read_pipe_lines(c):
  50    if verbose:
  51        sys.stderr.write('Reading pipe: %s\n' % c)
  52    ## todo: check return status
  53    pipe = os.popen(c, 'rb')
  54    val = pipe.readlines()
  55    if pipe.close():
  56        die('Command failed: %s' % c)
  57
  58    return val
  59
  60def system(cmd):
  61    if verbose:
  62        sys.stderr.write("executing %s\n" % cmd)
  63    if os.system(cmd) != 0:
  64        die("command failed: %s" % cmd)
  65
  66def isP4Exec(kind):
  67    """Determine if a Perforce 'kind' should have execute permission
  68
  69    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70    or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71    a plus sign, it is also executable"""
  72    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74def setP4ExecBit(file, mode):
  75    # Reopens an already open file and changes the execute bit to match
  76    # the execute bit setting in the passed in mode.
  77
  78    p4Type = "+x"
  79
  80    if not isModeExec(mode):
  81        p4Type = getP4OpenedType(file)
  82        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84        if p4Type[-1] == "+":
  85            p4Type = p4Type[0:-1]
  86
  87    system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89def getP4OpenedType(file):
  90    # Returns the perforce file type for the given file.
  91
  92    result = read_pipe("p4 opened %s" % file)
  93    match = re.match(".*\((.+)\)$", result)
  94    if match:
  95        return match.group(1)
  96    else:
  97        die("Could not determine file type for %s" % file)
  98
  99def diffTreePattern():
 100    # This is a simple generator for the diff tree regex pattern. This could be
 101    # a class variable if this and parseDiffTreeEntry were a part of a class.
 102    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103    while True:
 104        yield pattern
 105
 106def parseDiffTreeEntry(entry):
 107    """Parses a single diff tree entry into its component elements.
 108
 109    See git-diff-tree(1) manpage for details about the format of the diff
 110    output. This method returns a dictionary with the following elements:
 111
 112    src_mode - The mode of the source file
 113    dst_mode - The mode of the destination file
 114    src_sha1 - The sha1 for the source file
 115    dst_sha1 - The sha1 fr the destination file
 116    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117    status_score - The score for the status (applicable for 'C' and 'R'
 118                   statuses). This is None if there is no score.
 119    src - The path for the source file.
 120    dst - The path for the destination file. This is only present for
 121          copy or renames. If it is not present, this is None.
 122
 123    If the pattern is not matched, None is returned."""
 124
 125    match = diffTreePattern().next().match(entry)
 126    if match:
 127        return {
 128            'src_mode': match.group(1),
 129            'dst_mode': match.group(2),
 130            'src_sha1': match.group(3),
 131            'dst_sha1': match.group(4),
 132            'status': match.group(5),
 133            'status_score': match.group(6),
 134            'src': match.group(7),
 135            'dst': match.group(10)
 136        }
 137    return None
 138
 139def isModeExec(mode):
 140    # Returns True if the given git mode represents an executable file,
 141    # otherwise False.
 142    return mode[-3:] == "755"
 143
 144def isModeExecChanged(src_mode, dst_mode):
 145    return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148    cmd = "p4 -G %s" % cmd
 149    if verbose:
 150        sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152    # Use a temporary file to avoid deadlocks without
 153    # subprocess.communicate(), which would put another copy
 154    # of stdout into memory.
 155    stdin_file = None
 156    if stdin is not None:
 157        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158        stdin_file.write(stdin)
 159        stdin_file.flush()
 160        stdin_file.seek(0)
 161
 162    p4 = subprocess.Popen(cmd, shell=True,
 163                          stdin=stdin_file,
 164                          stdout=subprocess.PIPE)
 165
 166    result = []
 167    try:
 168        while True:
 169            entry = marshal.load(p4.stdout)
 170            result.append(entry)
 171    except EOFError:
 172        pass
 173    exitCode = p4.wait()
 174    if exitCode != 0:
 175        entry = {}
 176        entry["p4ExitCode"] = exitCode
 177        result.append(entry)
 178
 179    return result
 180
 181def p4Cmd(cmd):
 182    list = p4CmdList(cmd)
 183    result = {}
 184    for entry in list:
 185        result.update(entry)
 186    return result;
 187
 188def p4Where(depotPath):
 189    if not depotPath.endswith("/"):
 190        depotPath += "/"
 191    output = p4Cmd("where %s..." % depotPath)
 192    if output["code"] == "error":
 193        return ""
 194    clientPath = ""
 195    if "path" in output:
 196        clientPath = output.get("path")
 197    elif "data" in output:
 198        data = output.get("data")
 199        lastSpace = data.rfind(" ")
 200        clientPath = data[lastSpace + 1:]
 201
 202    if clientPath.endswith("..."):
 203        clientPath = clientPath[:-3]
 204    return clientPath
 205
 206def currentGitBranch():
 207    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209def isValidGitDir(path):
 210    if (os.path.exists(path + "/HEAD")
 211        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212        return True;
 213    return False
 214
 215def parseRevision(ref):
 216    return read_pipe("git rev-parse %s" % ref).strip()
 217
 218def extractLogMessageFromGitCommit(commit):
 219    logMessage = ""
 220
 221    ## fixme: title is first line of commit, not 1st paragraph.
 222    foundTitle = False
 223    for log in read_pipe_lines("git cat-file commit %s" % commit):
 224       if not foundTitle:
 225           if len(log) == 1:
 226               foundTitle = True
 227           continue
 228
 229       logMessage += log
 230    return logMessage
 231
 232def extractSettingsGitLog(log):
 233    values = {}
 234    for line in log.split("\n"):
 235        line = line.strip()
 236        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237        if not m:
 238            continue
 239
 240        assignments = m.group(1).split (':')
 241        for a in assignments:
 242            vals = a.split ('=')
 243            key = vals[0].strip()
 244            val = ('='.join (vals[1:])).strip()
 245            if val.endswith ('\"') and val.startswith('"'):
 246                val = val[1:-1]
 247
 248            values[key] = val
 249
 250    paths = values.get("depot-paths")
 251    if not paths:
 252        paths = values.get("depot-path")
 253    if paths:
 254        values['depot-paths'] = paths.split(',')
 255    return values
 256
 257def gitBranchExists(branch):
 258    proc = subprocess.Popen(["git", "rev-parse", branch],
 259                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260    return proc.wait() == 0;
 261
 262def gitConfig(key):
 263    return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265def p4BranchesInGit(branchesAreInRemotes = True):
 266    branches = {}
 267
 268    cmdline = "git rev-parse --symbolic "
 269    if branchesAreInRemotes:
 270        cmdline += " --remotes"
 271    else:
 272        cmdline += " --branches"
 273
 274    for line in read_pipe_lines(cmdline):
 275        line = line.strip()
 276
 277        ## only import to p4/
 278        if not line.startswith('p4/') or line == "p4/HEAD":
 279            continue
 280        branch = line
 281
 282        # strip off p4
 283        branch = re.sub ("^p4/", "", line)
 284
 285        branches[branch] = parseRevision(line)
 286    return branches
 287
 288def findUpstreamBranchPoint(head = "HEAD"):
 289    branches = p4BranchesInGit()
 290    # map from depot-path to branch name
 291    branchByDepotPath = {}
 292    for branch in branches.keys():
 293        tip = branches[branch]
 294        log = extractLogMessageFromGitCommit(tip)
 295        settings = extractSettingsGitLog(log)
 296        if settings.has_key("depot-paths"):
 297            paths = ",".join(settings["depot-paths"])
 298            branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300    settings = None
 301    parent = 0
 302    while parent < 65535:
 303        commit = head + "~%s" % parent
 304        log = extractLogMessageFromGitCommit(commit)
 305        settings = extractSettingsGitLog(log)
 306        if settings.has_key("depot-paths"):
 307            paths = ",".join(settings["depot-paths"])
 308            if branchByDepotPath.has_key(paths):
 309                return [branchByDepotPath[paths], settings]
 310
 311        parent = parent + 1
 312
 313    return ["", settings]
 314
 315def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316    if not silent:
 317        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318               % localRefPrefix)
 319
 320    originPrefix = "origin/p4/"
 321
 322    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323        line = line.strip()
 324        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325            continue
 326
 327        headName = line[len(originPrefix):]
 328        remoteHead = localRefPrefix + headName
 329        originHead = line
 330
 331        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332        if (not original.has_key('depot-paths')
 333            or not original.has_key('change')):
 334            continue
 335
 336        update = False
 337        if not gitBranchExists(remoteHead):
 338            if verbose:
 339                print "creating %s" % remoteHead
 340            update = True
 341        else:
 342            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343            if settings.has_key('change') > 0:
 344                if settings['depot-paths'] == original['depot-paths']:
 345                    originP4Change = int(original['change'])
 346                    p4Change = int(settings['change'])
 347                    if originP4Change > p4Change:
 348                        print ("%s (%s) is newer than %s (%s). "
 349                               "Updating p4 branch from origin."
 350                               % (originHead, originP4Change,
 351                                  remoteHead, p4Change))
 352                        update = True
 353                else:
 354                    print ("Ignoring: %s was imported from %s while "
 355                           "%s was imported from %s"
 356                           % (originHead, ','.join(original['depot-paths']),
 357                              remoteHead, ','.join(settings['depot-paths'])))
 358
 359        if update:
 360            system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362def originP4BranchesExist():
 363        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365def p4ChangesForPaths(depotPaths, changeRange):
 366    assert depotPaths
 367    output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                        for p in depotPaths]))
 369
 370    changes = []
 371    for line in output:
 372        changeNum = line.split(" ")[1]
 373        changes.append(int(changeNum))
 374
 375    changes.sort()
 376    return changes
 377
 378class Command:
 379    def __init__(self):
 380        self.usage = "usage: %prog [options]"
 381        self.needsGit = True
 382
 383class P4Debug(Command):
 384    def __init__(self):
 385        Command.__init__(self)
 386        self.options = [
 387            optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                 default=False),
 389            ]
 390        self.description = "A tool to debug the output of p4 -G."
 391        self.needsGit = False
 392        self.verbose = False
 393
 394    def run(self, args):
 395        j = 0
 396        for output in p4CmdList(" ".join(args)):
 397            print 'Element: %d' % j
 398            j += 1
 399            print output
 400        return True
 401
 402class P4RollBack(Command):
 403    def __init__(self):
 404        Command.__init__(self)
 405        self.options = [
 406            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408        ]
 409        self.description = "A tool to debug the multi-branch import. Don't use :)"
 410        self.verbose = False
 411        self.rollbackLocalBranches = False
 412
 413    def run(self, args):
 414        if len(args) != 1:
 415            return False
 416        maxChange = int(args[0])
 417
 418        if "p4ExitCode" in p4Cmd("changes -m 1"):
 419            die("Problems executing p4");
 420
 421        if self.rollbackLocalBranches:
 422            refPrefix = "refs/heads/"
 423            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424        else:
 425            refPrefix = "refs/remotes/"
 426            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428        for line in lines:
 429            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                line = line.strip()
 431                ref = refPrefix + line
 432                log = extractLogMessageFromGitCommit(ref)
 433                settings = extractSettingsGitLog(log)
 434
 435                depotPaths = settings['depot-paths']
 436                change = settings['change']
 437
 438                changed = False
 439
 440                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                           for p in depotPaths]))) == 0:
 442                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                    continue
 445
 446                while change and int(change) > maxChange:
 447                    changed = True
 448                    if self.verbose:
 449                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                    system("git update-ref %s \"%s^\"" % (ref, ref))
 451                    log = extractLogMessageFromGitCommit(ref)
 452                    settings =  extractSettingsGitLog(log)
 453
 454
 455                    depotPaths = settings['depot-paths']
 456                    change = settings['change']
 457
 458                if changed:
 459                    print "%s rewound to %s" % (ref, change)
 460
 461        return True
 462
 463class P4Submit(Command):
 464    def __init__(self):
 465        Command.__init__(self)
 466        self.options = [
 467                optparse.make_option("--continue", action="store_false", dest="firstTime"),
 468                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 469                optparse.make_option("--origin", dest="origin"),
 470                optparse.make_option("--reset", action="store_true", dest="reset"),
 471                optparse.make_option("--log-substitutions", dest="substFile"),
 472                optparse.make_option("--direct", dest="directSubmit", action="store_true"),
 473                optparse.make_option("-M", dest="detectRename", action="store_true"),
 474        ]
 475        self.description = "Submit changes from git to the perforce depot."
 476        self.usage += " [name of git branch to submit into perforce depot]"
 477        self.firstTime = True
 478        self.reset = False
 479        self.interactive = True
 480        self.substFile = ""
 481        self.firstTime = True
 482        self.origin = ""
 483        self.directSubmit = False
 484        self.detectRename = False
 485        self.verbose = False
 486        self.isWindows = (platform.system() == "Windows")
 487
 488        self.logSubstitutions = {}
 489        self.logSubstitutions["<enter description here>"] = "%log%"
 490        self.logSubstitutions["\tDetails:"] = "\tDetails:  %log%"
 491
 492    def check(self):
 493        if len(p4CmdList("opened ...")) > 0:
 494            die("You have files opened with perforce! Close them before starting the sync.")
 495
 496    def start(self):
 497        if len(self.config) > 0 and not self.reset:
 498            die("Cannot start sync. Previous sync config found at %s\n"
 499                "If you want to start submitting again from scratch "
 500                "maybe you want to call git-p4 submit --reset" % self.configFile)
 501
 502        commits = []
 503        if self.directSubmit:
 504            commits.append("0")
 505        else:
 506            for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 507                commits.append(line.strip())
 508            commits.reverse()
 509
 510        self.config["commits"] = commits
 511
 512    def prepareLogMessage(self, template, message):
 513        result = ""
 514
 515        for line in template.split("\n"):
 516            if line.startswith("#"):
 517                result += line + "\n"
 518                continue
 519
 520            substituted = False
 521            for key in self.logSubstitutions.keys():
 522                if line.find(key) != -1:
 523                    value = self.logSubstitutions[key]
 524                    value = value.replace("%log%", message)
 525                    if value != "@remove@":
 526                        result += line.replace(key, value) + "\n"
 527                    substituted = True
 528                    break
 529
 530            if not substituted:
 531                result += line + "\n"
 532
 533        return result
 534
 535    def prepareSubmitTemplate(self):
 536        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 537        template = ""
 538        inFilesSection = False
 539        for line in read_pipe_lines("p4 change -o"):
 540            if inFilesSection:
 541                if line.startswith("\t"):
 542                    # path starts and ends with a tab
 543                    path = line[1:]
 544                    lastTab = path.rfind("\t")
 545                    if lastTab != -1:
 546                        path = path[:lastTab]
 547                        if not path.startswith(self.depotPath):
 548                            continue
 549                else:
 550                    inFilesSection = False
 551            else:
 552                if line.startswith("Files:"):
 553                    inFilesSection = True
 554
 555            template += line
 556
 557        return template
 558
 559    def applyCommit(self, id):
 560        if self.directSubmit:
 561            print "Applying local change in working directory/index"
 562            diff = self.diffStatus
 563        else:
 564            print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 565            diffOpts = ("", "-M")[self.detectRename]
 566            diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 567        filesToAdd = set()
 568        filesToDelete = set()
 569        editedFiles = set()
 570        filesToChangeExecBit = {}
 571        for line in diff:
 572            diff = parseDiffTreeEntry(line)
 573            modifier = diff['status']
 574            path = diff['src']
 575            if modifier == "M":
 576                system("p4 edit \"%s\"" % path)
 577                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 578                    filesToChangeExecBit[path] = diff['dst_mode']
 579                editedFiles.add(path)
 580            elif modifier == "A":
 581                filesToAdd.add(path)
 582                filesToChangeExecBit[path] = diff['dst_mode']
 583                if path in filesToDelete:
 584                    filesToDelete.remove(path)
 585            elif modifier == "D":
 586                filesToDelete.add(path)
 587                if path in filesToAdd:
 588                    filesToAdd.remove(path)
 589            elif modifier == "R":
 590                src, dest = diff['src'], diff['dst']
 591                system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 592                system("p4 edit \"%s\"" % (dest))
 593                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 594                    filesToChangeExecBit[dest] = diff['dst_mode']
 595                os.unlink(dest)
 596                editedFiles.add(dest)
 597                filesToDelete.add(src)
 598            else:
 599                die("unknown modifier %s for %s" % (modifier, path))
 600
 601        if self.directSubmit:
 602            diffcmd = "cat \"%s\"" % self.diffFile
 603        else:
 604            diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 605        patchcmd = diffcmd + " | git apply "
 606        tryPatchCmd = patchcmd + "--check -"
 607        applyPatchCmd = patchcmd + "--check --apply -"
 608
 609        if os.system(tryPatchCmd) != 0:
 610            print "Unfortunately applying the change failed!"
 611            print "What do you want to do?"
 612            response = "x"
 613            while response != "s" and response != "a" and response != "w":
 614                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 615                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 616            if response == "s":
 617                print "Skipping! Good luck with the next patches..."
 618                for f in editedFiles:
 619                    system("p4 revert \"%s\"" % f);
 620                for f in filesToAdd:
 621                    system("rm %s" %f)
 622                return
 623            elif response == "a":
 624                os.system(applyPatchCmd)
 625                if len(filesToAdd) > 0:
 626                    print "You may also want to call p4 add on the following files:"
 627                    print " ".join(filesToAdd)
 628                if len(filesToDelete):
 629                    print "The following files should be scheduled for deletion with p4 delete:"
 630                    print " ".join(filesToDelete)
 631                die("Please resolve and submit the conflict manually and "
 632                    + "continue afterwards with git-p4 submit --continue")
 633            elif response == "w":
 634                system(diffcmd + " > patch.txt")
 635                print "Patch saved to patch.txt in %s !" % self.clientPath
 636                die("Please resolve and submit the conflict manually and "
 637                    "continue afterwards with git-p4 submit --continue")
 638
 639        system(applyPatchCmd)
 640
 641        for f in filesToAdd:
 642            system("p4 add \"%s\"" % f)
 643        for f in filesToDelete:
 644            system("p4 revert \"%s\"" % f)
 645            system("p4 delete \"%s\"" % f)
 646
 647        # Set/clear executable bits
 648        for f in filesToChangeExecBit.keys():
 649            mode = filesToChangeExecBit[f]
 650            setP4ExecBit(f, mode)
 651
 652        logMessage = ""
 653        if not self.directSubmit:
 654            logMessage = extractLogMessageFromGitCommit(id)
 655            logMessage = logMessage.replace("\n", "\n\t")
 656            if self.isWindows:
 657                logMessage = logMessage.replace("\n", "\r\n")
 658            logMessage = logMessage.strip()
 659
 660        template = self.prepareSubmitTemplate()
 661
 662        if self.interactive:
 663            submitTemplate = self.prepareLogMessage(template, logMessage)
 664            diff = read_pipe("p4 diff -du ...")
 665
 666            for newFile in filesToAdd:
 667                diff += "==== new file ====\n"
 668                diff += "--- /dev/null\n"
 669                diff += "+++ %s\n" % newFile
 670                f = open(newFile, "r")
 671                for line in f.readlines():
 672                    diff += "+" + line
 673                f.close()
 674
 675            separatorLine = "######## everything below this line is just the diff #######"
 676            if platform.system() == "Windows":
 677                separatorLine += "\r"
 678            separatorLine += "\n"
 679
 680            [handle, fileName] = tempfile.mkstemp()
 681            tmpFile = os.fdopen(handle, "w+")
 682            tmpFile.write(submitTemplate + separatorLine + diff)
 683            tmpFile.close()
 684            defaultEditor = "vi"
 685            if platform.system() == "Windows":
 686                defaultEditor = "notepad"
 687            editor = os.environ.get("EDITOR", defaultEditor);
 688            system(editor + " " + fileName)
 689            tmpFile = open(fileName, "rb")
 690            message = tmpFile.read()
 691            tmpFile.close()
 692            os.remove(fileName)
 693            submitTemplate = message[:message.index(separatorLine)]
 694            if self.isWindows:
 695                submitTemplate = submitTemplate.replace("\r\n", "\n")
 696
 697            if self.directSubmit:
 698                print "Submitting to git first"
 699                os.chdir(self.oldWorkingDirectory)
 700                write_pipe("git commit -a -F -", submitTemplate)
 701                os.chdir(self.clientPath)
 702
 703            write_pipe("p4 submit -i", submitTemplate)
 704        else:
 705            fileName = "submit.txt"
 706            file = open(fileName, "w+")
 707            file.write(self.prepareLogMessage(template, logMessage))
 708            file.close()
 709            print ("Perforce submit template written as %s. "
 710                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 711                   % (fileName, fileName))
 712
 713    def run(self, args):
 714        if len(args) == 0:
 715            self.master = currentGitBranch()
 716            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 717                die("Detecting current git branch failed!")
 718        elif len(args) == 1:
 719            self.master = args[0]
 720        else:
 721            return False
 722
 723        [upstream, settings] = findUpstreamBranchPoint()
 724        self.depotPath = settings['depot-paths'][0]
 725        if len(self.origin) == 0:
 726            self.origin = upstream
 727
 728        if self.verbose:
 729            print "Origin branch is " + self.origin
 730
 731        if len(self.depotPath) == 0:
 732            print "Internal error: cannot locate perforce depot path from existing branches"
 733            sys.exit(128)
 734
 735        self.clientPath = p4Where(self.depotPath)
 736
 737        if len(self.clientPath) == 0:
 738            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 739            sys.exit(128)
 740
 741        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 742        self.oldWorkingDirectory = os.getcwd()
 743
 744        if self.directSubmit:
 745            self.diffStatus = read_pipe_lines("git diff -r --name-status HEAD")
 746            if len(self.diffStatus) == 0:
 747                print "No changes in working directory to submit."
 748                return True
 749            patch = read_pipe("git diff -p --binary --diff-filter=ACMRTUXB HEAD")
 750            self.diffFile = self.gitdir + "/p4-git-diff"
 751            f = open(self.diffFile, "wb")
 752            f.write(patch)
 753            f.close();
 754
 755        os.chdir(self.clientPath)
 756        print "Syncronizing p4 checkout..."
 757        system("p4 sync ...")
 758
 759        if self.reset:
 760            self.firstTime = True
 761
 762        if len(self.substFile) > 0:
 763            for line in open(self.substFile, "r").readlines():
 764                tokens = line.strip().split("=")
 765                self.logSubstitutions[tokens[0]] = tokens[1]
 766
 767        self.check()
 768        self.configFile = self.gitdir + "/p4-git-sync.cfg"
 769        self.config = shelve.open(self.configFile, writeback=True)
 770
 771        if self.firstTime:
 772            self.start()
 773
 774        commits = self.config.get("commits", [])
 775
 776        while len(commits) > 0:
 777            self.firstTime = False
 778            commit = commits[0]
 779            commits = commits[1:]
 780            self.config["commits"] = commits
 781            self.applyCommit(commit)
 782            if not self.interactive:
 783                break
 784
 785        self.config.close()
 786
 787        if self.directSubmit:
 788            os.remove(self.diffFile)
 789
 790        if len(commits) == 0:
 791            if self.firstTime:
 792                print "No changes found to apply between %s and current HEAD" % self.origin
 793            else:
 794                print "All changes applied!"
 795                os.chdir(self.oldWorkingDirectory)
 796
 797                sync = P4Sync()
 798                sync.run([])
 799
 800                rebase = P4Rebase()
 801                rebase.rebase()
 802            os.remove(self.configFile)
 803
 804        return True
 805
 806class P4Sync(Command):
 807    def __init__(self):
 808        Command.__init__(self)
 809        self.options = [
 810                optparse.make_option("--branch", dest="branch"),
 811                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 812                optparse.make_option("--changesfile", dest="changesFile"),
 813                optparse.make_option("--silent", dest="silent", action="store_true"),
 814                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 815                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 816                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 817                                     help="Import into refs/heads/ , not refs/remotes"),
 818                optparse.make_option("--max-changes", dest="maxChanges"),
 819                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 820                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import")
 821        ]
 822        self.description = """Imports from Perforce into a git repository.\n
 823    example:
 824    //depot/my/project/ -- to import the current head
 825    //depot/my/project/@all -- to import everything
 826    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 827
 828    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 829
 830        self.usage += " //depot/path[@revRange]"
 831        self.silent = False
 832        self.createdBranches = Set()
 833        self.committedChanges = Set()
 834        self.branch = ""
 835        self.detectBranches = False
 836        self.detectLabels = False
 837        self.changesFile = ""
 838        self.syncWithOrigin = True
 839        self.verbose = False
 840        self.importIntoRemotes = True
 841        self.maxChanges = ""
 842        self.isWindows = (platform.system() == "Windows")
 843        self.keepRepoPath = False
 844        self.depotPaths = None
 845        self.p4BranchesInGit = []
 846
 847        if gitConfig("git-p4.syncFromOrigin") == "false":
 848            self.syncWithOrigin = False
 849
 850    def extractFilesFromCommit(self, commit):
 851        files = []
 852        fnum = 0
 853        while commit.has_key("depotFile%s" % fnum):
 854            path =  commit["depotFile%s" % fnum]
 855
 856            found = [p for p in self.depotPaths
 857                     if path.startswith (p)]
 858            if not found:
 859                fnum = fnum + 1
 860                continue
 861
 862            file = {}
 863            file["path"] = path
 864            file["rev"] = commit["rev%s" % fnum]
 865            file["action"] = commit["action%s" % fnum]
 866            file["type"] = commit["type%s" % fnum]
 867            files.append(file)
 868            fnum = fnum + 1
 869        return files
 870
 871    def stripRepoPath(self, path, prefixes):
 872        if self.keepRepoPath:
 873            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 874
 875        for p in prefixes:
 876            if path.startswith(p):
 877                path = path[len(p):]
 878
 879        return path
 880
 881    def splitFilesIntoBranches(self, commit):
 882        branches = {}
 883        fnum = 0
 884        while commit.has_key("depotFile%s" % fnum):
 885            path =  commit["depotFile%s" % fnum]
 886            found = [p for p in self.depotPaths
 887                     if path.startswith (p)]
 888            if not found:
 889                fnum = fnum + 1
 890                continue
 891
 892            file = {}
 893            file["path"] = path
 894            file["rev"] = commit["rev%s" % fnum]
 895            file["action"] = commit["action%s" % fnum]
 896            file["type"] = commit["type%s" % fnum]
 897            fnum = fnum + 1
 898
 899            relPath = self.stripRepoPath(path, self.depotPaths)
 900
 901            for branch in self.knownBranches.keys():
 902
 903                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 904                if relPath.startswith(branch + "/"):
 905                    if branch not in branches:
 906                        branches[branch] = []
 907                    branches[branch].append(file)
 908                    break
 909
 910        return branches
 911
 912    ## Should move this out, doesn't use SELF.
 913    def readP4Files(self, files):
 914        files = [f for f in files
 915                 if f['action'] != 'delete']
 916
 917        if not files:
 918            return
 919
 920        filedata = p4CmdList('-x - print',
 921                             stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 922                                              for f in files]),
 923                             stdin_mode='w+')
 924        if "p4ExitCode" in filedata[0]:
 925            die("Problems executing p4. Error: [%d]."
 926                % (filedata[0]['p4ExitCode']));
 927
 928        j = 0;
 929        contents = {}
 930        while j < len(filedata):
 931            stat = filedata[j]
 932            j += 1
 933            text = ''
 934            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 935                tmp = filedata[j]['data']
 936                if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 937                    tmp = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', tmp)
 938                elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 939                    tmp = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', tmp)
 940                text += tmp
 941                j += 1
 942
 943
 944            if not stat.has_key('depotFile'):
 945                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 946                continue
 947
 948            contents[stat['depotFile']] = text
 949
 950        for f in files:
 951            assert not f.has_key('data')
 952            f['data'] = contents[f['path']]
 953
 954    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 955        epoch = details["time"]
 956        author = details["user"]
 957
 958        if self.verbose:
 959            print "commit into %s" % branch
 960
 961        # start with reading files; if that fails, we should not
 962        # create a commit.
 963        new_files = []
 964        for f in files:
 965            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 966                new_files.append (f)
 967            else:
 968                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 969        files = new_files
 970        self.readP4Files(files)
 971
 972
 973
 974
 975        self.gitStream.write("commit %s\n" % branch)
 976#        gitStream.write("mark :%s\n" % details["change"])
 977        self.committedChanges.add(int(details["change"]))
 978        committer = ""
 979        if author not in self.users:
 980            self.getUserMapFromPerforceServer()
 981        if author in self.users:
 982            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 983        else:
 984            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 985
 986        self.gitStream.write("committer %s\n" % committer)
 987
 988        self.gitStream.write("data <<EOT\n")
 989        self.gitStream.write(details["desc"])
 990        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 991                             % (','.join (branchPrefixes), details["change"]))
 992        if len(details['options']) > 0:
 993            self.gitStream.write(": options = %s" % details['options'])
 994        self.gitStream.write("]\nEOT\n\n")
 995
 996        if len(parent) > 0:
 997            if self.verbose:
 998                print "parent %s" % parent
 999            self.gitStream.write("from %s\n" % parent)
1000
1001        for file in files:
1002            if file["type"] == "apple":
1003                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1004                continue
1005
1006            relPath = self.stripRepoPath(file['path'], branchPrefixes)
1007            if file["action"] == "delete":
1008                self.gitStream.write("D %s\n" % relPath)
1009            else:
1010                data = file['data']
1011
1012                mode = "644"
1013                if isP4Exec(file["type"]):
1014                    mode = "755"
1015                elif file["type"] == "symlink":
1016                    mode = "120000"
1017                    # p4 print on a symlink contains "target\n", so strip it off
1018                    data = data[:-1]
1019
1020                if self.isWindows and file["type"].endswith("text"):
1021                    data = data.replace("\r\n", "\n")
1022
1023                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1024                self.gitStream.write("data %s\n" % len(data))
1025                self.gitStream.write(data)
1026                self.gitStream.write("\n")
1027
1028        self.gitStream.write("\n")
1029
1030        change = int(details["change"])
1031
1032        if self.labels.has_key(change):
1033            label = self.labels[change]
1034            labelDetails = label[0]
1035            labelRevisions = label[1]
1036            if self.verbose:
1037                print "Change %s is labelled %s" % (change, labelDetails)
1038
1039            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1040                                                    for p in branchPrefixes]))
1041
1042            if len(files) == len(labelRevisions):
1043
1044                cleanedFiles = {}
1045                for info in files:
1046                    if info["action"] == "delete":
1047                        continue
1048                    cleanedFiles[info["depotFile"]] = info["rev"]
1049
1050                if cleanedFiles == labelRevisions:
1051                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1052                    self.gitStream.write("from %s\n" % branch)
1053
1054                    owner = labelDetails["Owner"]
1055                    tagger = ""
1056                    if author in self.users:
1057                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1058                    else:
1059                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1060                    self.gitStream.write("tagger %s\n" % tagger)
1061                    self.gitStream.write("data <<EOT\n")
1062                    self.gitStream.write(labelDetails["Description"])
1063                    self.gitStream.write("EOT\n\n")
1064
1065                else:
1066                    if not self.silent:
1067                        print ("Tag %s does not match with change %s: files do not match."
1068                               % (labelDetails["label"], change))
1069
1070            else:
1071                if not self.silent:
1072                    print ("Tag %s does not match with change %s: file count is different."
1073                           % (labelDetails["label"], change))
1074
1075    def getUserCacheFilename(self):
1076        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1077        return home + "/.gitp4-usercache.txt"
1078
1079    def getUserMapFromPerforceServer(self):
1080        if self.userMapFromPerforceServer:
1081            return
1082        self.users = {}
1083
1084        for output in p4CmdList("users"):
1085            if not output.has_key("User"):
1086                continue
1087            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1088
1089
1090        s = ''
1091        for (key, val) in self.users.items():
1092            s += "%s\t%s\n" % (key, val)
1093
1094        open(self.getUserCacheFilename(), "wb").write(s)
1095        self.userMapFromPerforceServer = True
1096
1097    def loadUserMapFromCache(self):
1098        self.users = {}
1099        self.userMapFromPerforceServer = False
1100        try:
1101            cache = open(self.getUserCacheFilename(), "rb")
1102            lines = cache.readlines()
1103            cache.close()
1104            for line in lines:
1105                entry = line.strip().split("\t")
1106                self.users[entry[0]] = entry[1]
1107        except IOError:
1108            self.getUserMapFromPerforceServer()
1109
1110    def getLabels(self):
1111        self.labels = {}
1112
1113        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1114        if len(l) > 0 and not self.silent:
1115            print "Finding files belonging to labels in %s" % `self.depotPaths`
1116
1117        for output in l:
1118            label = output["label"]
1119            revisions = {}
1120            newestChange = 0
1121            if self.verbose:
1122                print "Querying files for label %s" % label
1123            for file in p4CmdList("files "
1124                                  +  ' '.join (["%s...@%s" % (p, label)
1125                                                for p in self.depotPaths])):
1126                revisions[file["depotFile"]] = file["rev"]
1127                change = int(file["change"])
1128                if change > newestChange:
1129                    newestChange = change
1130
1131            self.labels[newestChange] = [output, revisions]
1132
1133        if self.verbose:
1134            print "Label changes: %s" % self.labels.keys()
1135
1136    def guessProjectName(self):
1137        for p in self.depotPaths:
1138            if p.endswith("/"):
1139                p = p[:-1]
1140            p = p[p.strip().rfind("/") + 1:]
1141            if not p.endswith("/"):
1142               p += "/"
1143            return p
1144
1145    def getBranchMapping(self):
1146        lostAndFoundBranches = set()
1147
1148        for info in p4CmdList("branches"):
1149            details = p4Cmd("branch -o %s" % info["branch"])
1150            viewIdx = 0
1151            while details.has_key("View%s" % viewIdx):
1152                paths = details["View%s" % viewIdx].split(" ")
1153                viewIdx = viewIdx + 1
1154                # require standard //depot/foo/... //depot/bar/... mapping
1155                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1156                    continue
1157                source = paths[0]
1158                destination = paths[1]
1159                ## HACK
1160                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1161                    source = source[len(self.depotPaths[0]):-4]
1162                    destination = destination[len(self.depotPaths[0]):-4]
1163
1164                    if destination in self.knownBranches:
1165                        if not self.silent:
1166                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1167                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1168                        continue
1169
1170                    self.knownBranches[destination] = source
1171
1172                    lostAndFoundBranches.discard(destination)
1173
1174                    if source not in self.knownBranches:
1175                        lostAndFoundBranches.add(source)
1176
1177
1178        for branch in lostAndFoundBranches:
1179            self.knownBranches[branch] = branch
1180
1181    def getBranchMappingFromGitBranches(self):
1182        branches = p4BranchesInGit(self.importIntoRemotes)
1183        for branch in branches.keys():
1184            if branch == "master":
1185                branch = "main"
1186            else:
1187                branch = branch[len(self.projectName):]
1188            self.knownBranches[branch] = branch
1189
1190    def listExistingP4GitBranches(self):
1191        # branches holds mapping from name to commit
1192        branches = p4BranchesInGit(self.importIntoRemotes)
1193        self.p4BranchesInGit = branches.keys()
1194        for branch in branches.keys():
1195            self.initialParents[self.refPrefix + branch] = branches[branch]
1196
1197    def updateOptionDict(self, d):
1198        option_keys = {}
1199        if self.keepRepoPath:
1200            option_keys['keepRepoPath'] = 1
1201
1202        d["options"] = ' '.join(sorted(option_keys.keys()))
1203
1204    def readOptions(self, d):
1205        self.keepRepoPath = (d.has_key('options')
1206                             and ('keepRepoPath' in d['options']))
1207
1208    def gitRefForBranch(self, branch):
1209        if branch == "main":
1210            return self.refPrefix + "master"
1211
1212        if len(branch) <= 0:
1213            return branch
1214
1215        return self.refPrefix + self.projectName + branch
1216
1217    def gitCommitByP4Change(self, ref, change):
1218        if self.verbose:
1219            print "looking in ref " + ref + " for change %s using bisect..." % change
1220
1221        earliestCommit = ""
1222        latestCommit = parseRevision(ref)
1223
1224        while True:
1225            if self.verbose:
1226                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1227            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1228            if len(next) == 0:
1229                if self.verbose:
1230                    print "argh"
1231                return ""
1232            log = extractLogMessageFromGitCommit(next)
1233            settings = extractSettingsGitLog(log)
1234            currentChange = int(settings['change'])
1235            if self.verbose:
1236                print "current change %s" % currentChange
1237
1238            if currentChange == change:
1239                if self.verbose:
1240                    print "found %s" % next
1241                return next
1242
1243            if currentChange < change:
1244                earliestCommit = "^%s" % next
1245            else:
1246                latestCommit = "%s" % next
1247
1248        return ""
1249
1250    def importNewBranch(self, branch, maxChange):
1251        # make fast-import flush all changes to disk and update the refs using the checkpoint
1252        # command so that we can try to find the branch parent in the git history
1253        self.gitStream.write("checkpoint\n\n");
1254        self.gitStream.flush();
1255        branchPrefix = self.depotPaths[0] + branch + "/"
1256        range = "@1,%s" % maxChange
1257        #print "prefix" + branchPrefix
1258        changes = p4ChangesForPaths([branchPrefix], range)
1259        if len(changes) <= 0:
1260            return False
1261        firstChange = changes[0]
1262        #print "first change in branch: %s" % firstChange
1263        sourceBranch = self.knownBranches[branch]
1264        sourceDepotPath = self.depotPaths[0] + sourceBranch
1265        sourceRef = self.gitRefForBranch(sourceBranch)
1266        #print "source " + sourceBranch
1267
1268        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1269        #print "branch parent: %s" % branchParentChange
1270        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1271        if len(gitParent) > 0:
1272            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1273            #print "parent git commit: %s" % gitParent
1274
1275        self.importChanges(changes)
1276        return True
1277
1278    def importChanges(self, changes):
1279        cnt = 1
1280        for change in changes:
1281            description = p4Cmd("describe %s" % change)
1282            self.updateOptionDict(description)
1283
1284            if not self.silent:
1285                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1286                sys.stdout.flush()
1287            cnt = cnt + 1
1288
1289            try:
1290                if self.detectBranches:
1291                    branches = self.splitFilesIntoBranches(description)
1292                    for branch in branches.keys():
1293                        ## HACK  --hwn
1294                        branchPrefix = self.depotPaths[0] + branch + "/"
1295
1296                        parent = ""
1297
1298                        filesForCommit = branches[branch]
1299
1300                        if self.verbose:
1301                            print "branch is %s" % branch
1302
1303                        self.updatedBranches.add(branch)
1304
1305                        if branch not in self.createdBranches:
1306                            self.createdBranches.add(branch)
1307                            parent = self.knownBranches[branch]
1308                            if parent == branch:
1309                                parent = ""
1310                            else:
1311                                fullBranch = self.projectName + branch
1312                                if fullBranch not in self.p4BranchesInGit:
1313                                    if not self.silent:
1314                                        print("\n    Importing new branch %s" % fullBranch);
1315                                    if self.importNewBranch(branch, change - 1):
1316                                        parent = ""
1317                                        self.p4BranchesInGit.append(fullBranch)
1318                                    if not self.silent:
1319                                        print("\n    Resuming with change %s" % change);
1320
1321                                if self.verbose:
1322                                    print "parent determined through known branches: %s" % parent
1323
1324                        branch = self.gitRefForBranch(branch)
1325                        parent = self.gitRefForBranch(parent)
1326
1327                        if self.verbose:
1328                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1329
1330                        if len(parent) == 0 and branch in self.initialParents:
1331                            parent = self.initialParents[branch]
1332                            del self.initialParents[branch]
1333
1334                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1335                else:
1336                    files = self.extractFilesFromCommit(description)
1337                    self.commit(description, files, self.branch, self.depotPaths,
1338                                self.initialParent)
1339                    self.initialParent = ""
1340            except IOError:
1341                print self.gitError.read()
1342                sys.exit(1)
1343
1344    def importHeadRevision(self, revision):
1345        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1346
1347        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1348        details["desc"] = ("Initial import of %s from the state at revision %s"
1349                           % (' '.join(self.depotPaths), revision))
1350        details["change"] = revision
1351        newestRevision = 0
1352
1353        fileCnt = 0
1354        for info in p4CmdList("files "
1355                              +  ' '.join(["%s...%s"
1356                                           % (p, revision)
1357                                           for p in self.depotPaths])):
1358
1359            if info['code'] == 'error':
1360                sys.stderr.write("p4 returned an error: %s\n"
1361                                 % info['data'])
1362                sys.exit(1)
1363
1364
1365            change = int(info["change"])
1366            if change > newestRevision:
1367                newestRevision = change
1368
1369            if info["action"] == "delete":
1370                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1371                #fileCnt = fileCnt + 1
1372                continue
1373
1374            for prop in ["depotFile", "rev", "action", "type" ]:
1375                details["%s%s" % (prop, fileCnt)] = info[prop]
1376
1377            fileCnt = fileCnt + 1
1378
1379        details["change"] = newestRevision
1380        self.updateOptionDict(details)
1381        try:
1382            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1383        except IOError:
1384            print "IO error with git fast-import. Is your git version recent enough?"
1385            print self.gitError.read()
1386
1387
1388    def run(self, args):
1389        self.depotPaths = []
1390        self.changeRange = ""
1391        self.initialParent = ""
1392        self.previousDepotPaths = []
1393
1394        # map from branch depot path to parent branch
1395        self.knownBranches = {}
1396        self.initialParents = {}
1397        self.hasOrigin = originP4BranchesExist()
1398        if not self.syncWithOrigin:
1399            self.hasOrigin = False
1400
1401        if self.importIntoRemotes:
1402            self.refPrefix = "refs/remotes/p4/"
1403        else:
1404            self.refPrefix = "refs/heads/p4/"
1405
1406        if self.syncWithOrigin and self.hasOrigin:
1407            if not self.silent:
1408                print "Syncing with origin first by calling git fetch origin"
1409            system("git fetch origin")
1410
1411        if len(self.branch) == 0:
1412            self.branch = self.refPrefix + "master"
1413            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1414                system("git update-ref %s refs/heads/p4" % self.branch)
1415                system("git branch -D p4");
1416            # create it /after/ importing, when master exists
1417            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1418                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1419
1420        # TODO: should always look at previous commits,
1421        # merge with previous imports, if possible.
1422        if args == []:
1423            if self.hasOrigin:
1424                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1425            self.listExistingP4GitBranches()
1426
1427            if len(self.p4BranchesInGit) > 1:
1428                if not self.silent:
1429                    print "Importing from/into multiple branches"
1430                self.detectBranches = True
1431
1432            if self.verbose:
1433                print "branches: %s" % self.p4BranchesInGit
1434
1435            p4Change = 0
1436            for branch in self.p4BranchesInGit:
1437                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1438
1439                settings = extractSettingsGitLog(logMsg)
1440
1441                self.readOptions(settings)
1442                if (settings.has_key('depot-paths')
1443                    and settings.has_key ('change')):
1444                    change = int(settings['change']) + 1
1445                    p4Change = max(p4Change, change)
1446
1447                    depotPaths = sorted(settings['depot-paths'])
1448                    if self.previousDepotPaths == []:
1449                        self.previousDepotPaths = depotPaths
1450                    else:
1451                        paths = []
1452                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1453                            for i in range(0, min(len(cur), len(prev))):
1454                                if cur[i] <> prev[i]:
1455                                    i = i - 1
1456                                    break
1457
1458                            paths.append (cur[:i + 1])
1459
1460                        self.previousDepotPaths = paths
1461
1462            if p4Change > 0:
1463                self.depotPaths = sorted(self.previousDepotPaths)
1464                self.changeRange = "@%s,#head" % p4Change
1465                if not self.detectBranches:
1466                    self.initialParent = parseRevision(self.branch)
1467                if not self.silent and not self.detectBranches:
1468                    print "Performing incremental import into %s git branch" % self.branch
1469
1470        if not self.branch.startswith("refs/"):
1471            self.branch = "refs/heads/" + self.branch
1472
1473        if len(args) == 0 and self.depotPaths:
1474            if not self.silent:
1475                print "Depot paths: %s" % ' '.join(self.depotPaths)
1476        else:
1477            if self.depotPaths and self.depotPaths != args:
1478                print ("previous import used depot path %s and now %s was specified. "
1479                       "This doesn't work!" % (' '.join (self.depotPaths),
1480                                               ' '.join (args)))
1481                sys.exit(1)
1482
1483            self.depotPaths = sorted(args)
1484
1485        revision = ""
1486        self.users = {}
1487
1488        newPaths = []
1489        for p in self.depotPaths:
1490            if p.find("@") != -1:
1491                atIdx = p.index("@")
1492                self.changeRange = p[atIdx:]
1493                if self.changeRange == "@all":
1494                    self.changeRange = ""
1495                elif ',' not in self.changeRange:
1496                    revision = self.changeRange
1497                    self.changeRange = ""
1498                p = p[:atIdx]
1499            elif p.find("#") != -1:
1500                hashIdx = p.index("#")
1501                revision = p[hashIdx:]
1502                p = p[:hashIdx]
1503            elif self.previousDepotPaths == []:
1504                revision = "#head"
1505
1506            p = re.sub ("\.\.\.$", "", p)
1507            if not p.endswith("/"):
1508                p += "/"
1509
1510            newPaths.append(p)
1511
1512        self.depotPaths = newPaths
1513
1514
1515        self.loadUserMapFromCache()
1516        self.labels = {}
1517        if self.detectLabels:
1518            self.getLabels();
1519
1520        if self.detectBranches:
1521            ## FIXME - what's a P4 projectName ?
1522            self.projectName = self.guessProjectName()
1523
1524            if self.hasOrigin:
1525                self.getBranchMappingFromGitBranches()
1526            else:
1527                self.getBranchMapping()
1528            if self.verbose:
1529                print "p4-git branches: %s" % self.p4BranchesInGit
1530                print "initial parents: %s" % self.initialParents
1531            for b in self.p4BranchesInGit:
1532                if b != "master":
1533
1534                    ## FIXME
1535                    b = b[len(self.projectName):]
1536                self.createdBranches.add(b)
1537
1538        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1539
1540        importProcess = subprocess.Popen(["git", "fast-import"],
1541                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1542                                         stderr=subprocess.PIPE);
1543        self.gitOutput = importProcess.stdout
1544        self.gitStream = importProcess.stdin
1545        self.gitError = importProcess.stderr
1546
1547        if revision:
1548            self.importHeadRevision(revision)
1549        else:
1550            changes = []
1551
1552            if len(self.changesFile) > 0:
1553                output = open(self.changesFile).readlines()
1554                changeSet = Set()
1555                for line in output:
1556                    changeSet.add(int(line))
1557
1558                for change in changeSet:
1559                    changes.append(change)
1560
1561                changes.sort()
1562            else:
1563                if self.verbose:
1564                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1565                                                              self.changeRange)
1566                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1567
1568                if len(self.maxChanges) > 0:
1569                    changes = changes[:min(int(self.maxChanges), len(changes))]
1570
1571            if len(changes) == 0:
1572                if not self.silent:
1573                    print "No changes to import!"
1574                return True
1575
1576            if not self.silent and not self.detectBranches:
1577                print "Import destination: %s" % self.branch
1578
1579            self.updatedBranches = set()
1580
1581            self.importChanges(changes)
1582
1583            if not self.silent:
1584                print ""
1585                if len(self.updatedBranches) > 0:
1586                    sys.stdout.write("Updated branches: ")
1587                    for b in self.updatedBranches:
1588                        sys.stdout.write("%s " % b)
1589                    sys.stdout.write("\n")
1590
1591        self.gitStream.close()
1592        if importProcess.wait() != 0:
1593            die("fast-import failed: %s" % self.gitError.read())
1594        self.gitOutput.close()
1595        self.gitError.close()
1596
1597        return True
1598
1599class P4Rebase(Command):
1600    def __init__(self):
1601        Command.__init__(self)
1602        self.options = [ ]
1603        self.description = ("Fetches the latest revision from perforce and "
1604                            + "rebases the current work (branch) against it")
1605        self.verbose = False
1606
1607    def run(self, args):
1608        sync = P4Sync()
1609        sync.run([])
1610
1611        return self.rebase()
1612
1613    def rebase(self):
1614        if os.system("git update-index --refresh") != 0:
1615            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1616        if len(read_pipe("git diff-index HEAD --")) > 0:
1617            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1618
1619        [upstream, settings] = findUpstreamBranchPoint()
1620        if len(upstream) == 0:
1621            die("Cannot find upstream branchpoint for rebase")
1622
1623        # the branchpoint may be p4/foo~3, so strip off the parent
1624        upstream = re.sub("~[0-9]+$", "", upstream)
1625
1626        print "Rebasing the current branch onto %s" % upstream
1627        oldHead = read_pipe("git rev-parse HEAD").strip()
1628        system("git rebase %s" % upstream)
1629        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1630        return True
1631
1632class P4Clone(P4Sync):
1633    def __init__(self):
1634        P4Sync.__init__(self)
1635        self.description = "Creates a new git repository and imports from Perforce into it"
1636        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1637        self.options.append(
1638            optparse.make_option("--destination", dest="cloneDestination",
1639                                 action='store', default=None,
1640                                 help="where to leave result of the clone"))
1641        self.cloneDestination = None
1642        self.needsGit = False
1643
1644    def defaultDestination(self, args):
1645        ## TODO: use common prefix of args?
1646        depotPath = args[0]
1647        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1648        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1649        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1650        depotDir = re.sub(r"/$", "", depotDir)
1651        return os.path.split(depotDir)[1]
1652
1653    def run(self, args):
1654        if len(args) < 1:
1655            return False
1656
1657        if self.keepRepoPath and not self.cloneDestination:
1658            sys.stderr.write("Must specify destination for --keep-path\n")
1659            sys.exit(1)
1660
1661        depotPaths = args
1662
1663        if not self.cloneDestination and len(depotPaths) > 1:
1664            self.cloneDestination = depotPaths[-1]
1665            depotPaths = depotPaths[:-1]
1666
1667        for p in depotPaths:
1668            if not p.startswith("//"):
1669                return False
1670
1671        if not self.cloneDestination:
1672            self.cloneDestination = self.defaultDestination(args)
1673
1674        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1675        if not os.path.exists(self.cloneDestination):
1676            os.makedirs(self.cloneDestination)
1677        os.chdir(self.cloneDestination)
1678        system("git init")
1679        self.gitdir = os.getcwd() + "/.git"
1680        if not P4Sync.run(self, depotPaths):
1681            return False
1682        if self.branch != "master":
1683            if gitBranchExists("refs/remotes/p4/master"):
1684                system("git branch master refs/remotes/p4/master")
1685                system("git checkout -f")
1686            else:
1687                print "Could not detect main branch. No checkout/master branch created."
1688
1689        return True
1690
1691class P4Branches(Command):
1692    def __init__(self):
1693        Command.__init__(self)
1694        self.options = [ ]
1695        self.description = ("Shows the git branches that hold imports and their "
1696                            + "corresponding perforce depot paths")
1697        self.verbose = False
1698
1699    def run(self, args):
1700        if originP4BranchesExist():
1701            createOrUpdateBranchesFromOrigin()
1702
1703        cmdline = "git rev-parse --symbolic "
1704        cmdline += " --remotes"
1705
1706        for line in read_pipe_lines(cmdline):
1707            line = line.strip()
1708
1709            if not line.startswith('p4/') or line == "p4/HEAD":
1710                continue
1711            branch = line
1712
1713            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1714            settings = extractSettingsGitLog(log)
1715
1716            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1717        return True
1718
1719class HelpFormatter(optparse.IndentedHelpFormatter):
1720    def __init__(self):
1721        optparse.IndentedHelpFormatter.__init__(self)
1722
1723    def format_description(self, description):
1724        if description:
1725            return description + "\n"
1726        else:
1727            return ""
1728
1729def printUsage(commands):
1730    print "usage: %s <command> [options]" % sys.argv[0]
1731    print ""
1732    print "valid commands: %s" % ", ".join(commands)
1733    print ""
1734    print "Try %s <command> --help for command specific help." % sys.argv[0]
1735    print ""
1736
1737commands = {
1738    "debug" : P4Debug,
1739    "submit" : P4Submit,
1740    "commit" : P4Submit,
1741    "sync" : P4Sync,
1742    "rebase" : P4Rebase,
1743    "clone" : P4Clone,
1744    "rollback" : P4RollBack,
1745    "branches" : P4Branches
1746}
1747
1748
1749def main():
1750    if len(sys.argv[1:]) == 0:
1751        printUsage(commands.keys())
1752        sys.exit(2)
1753
1754    cmd = ""
1755    cmdName = sys.argv[1]
1756    try:
1757        klass = commands[cmdName]
1758        cmd = klass()
1759    except KeyError:
1760        print "unknown command %s" % cmdName
1761        print ""
1762        printUsage(commands.keys())
1763        sys.exit(2)
1764
1765    options = cmd.options
1766    cmd.gitdir = os.environ.get("GIT_DIR", None)
1767
1768    args = sys.argv[2:]
1769
1770    if len(options) > 0:
1771        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1772
1773        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1774                                       options,
1775                                       description = cmd.description,
1776                                       formatter = HelpFormatter())
1777
1778        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1779    global verbose
1780    verbose = cmd.verbose
1781    if cmd.needsGit:
1782        if cmd.gitdir == None:
1783            cmd.gitdir = os.path.abspath(".git")
1784            if not isValidGitDir(cmd.gitdir):
1785                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1786                if os.path.exists(cmd.gitdir):
1787                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1788                    if len(cdup) > 0:
1789                        os.chdir(cdup);
1790
1791        if not isValidGitDir(cmd.gitdir):
1792            if isValidGitDir(cmd.gitdir + "/.git"):
1793                cmd.gitdir += "/.git"
1794            else:
1795                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1796
1797        os.environ["GIT_DIR"] = cmd.gitdir
1798
1799    if not cmd.run(args):
1800        parser.print_help()
1801
1802
1803if __name__ == '__main__':
1804    main()