Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit git-p4: support exclude paths (354081d)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19def die(msg):
  20    if verbose:
  21        raise Exception(msg)
  22    else:
  23        sys.stderr.write(msg + "\n")
  24        sys.exit(1)
  25
  26def write_pipe(c, str):
  27    if verbose:
  28        sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30    pipe = os.popen(c, 'w')
  31    val = pipe.write(str)
  32    if pipe.close():
  33        die('Command failed: %s' % c)
  34
  35    return val
  36
  37def read_pipe(c, ignore_error=False):
  38    if verbose:
  39        sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41    pipe = os.popen(c, 'rb')
  42    val = pipe.read()
  43    if pipe.close() and not ignore_error:
  44        die('Command failed: %s' % c)
  45
  46    return val
  47
  48
  49def read_pipe_lines(c):
  50    if verbose:
  51        sys.stderr.write('Reading pipe: %s\n' % c)
  52    ## todo: check return status
  53    pipe = os.popen(c, 'rb')
  54    val = pipe.readlines()
  55    if pipe.close():
  56        die('Command failed: %s' % c)
  57
  58    return val
  59
  60def system(cmd):
  61    if verbose:
  62        sys.stderr.write("executing %s\n" % cmd)
  63    if os.system(cmd) != 0:
  64        die("command failed: %s" % cmd)
  65
  66def isP4Exec(kind):
  67    """Determine if a Perforce 'kind' should have execute permission
  68
  69    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70    or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71    a plus sign, it is also executable"""
  72    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74def setP4ExecBit(file, mode):
  75    # Reopens an already open file and changes the execute bit to match
  76    # the execute bit setting in the passed in mode.
  77
  78    p4Type = "+x"
  79
  80    if not isModeExec(mode):
  81        p4Type = getP4OpenedType(file)
  82        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84        if p4Type[-1] == "+":
  85            p4Type = p4Type[0:-1]
  86
  87    system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89def getP4OpenedType(file):
  90    # Returns the perforce file type for the given file.
  91
  92    result = read_pipe("p4 opened %s" % file)
  93    match = re.match(".*\((.+)\)$", result)
  94    if match:
  95        return match.group(1)
  96    else:
  97        die("Could not determine file type for %s" % file)
  98
  99def diffTreePattern():
 100    # This is a simple generator for the diff tree regex pattern. This could be
 101    # a class variable if this and parseDiffTreeEntry were a part of a class.
 102    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103    while True:
 104        yield pattern
 105
 106def parseDiffTreeEntry(entry):
 107    """Parses a single diff tree entry into its component elements.
 108
 109    See git-diff-tree(1) manpage for details about the format of the diff
 110    output. This method returns a dictionary with the following elements:
 111
 112    src_mode - The mode of the source file
 113    dst_mode - The mode of the destination file
 114    src_sha1 - The sha1 for the source file
 115    dst_sha1 - The sha1 fr the destination file
 116    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117    status_score - The score for the status (applicable for 'C' and 'R'
 118                   statuses). This is None if there is no score.
 119    src - The path for the source file.
 120    dst - The path for the destination file. This is only present for
 121          copy or renames. If it is not present, this is None.
 122
 123    If the pattern is not matched, None is returned."""
 124
 125    match = diffTreePattern().next().match(entry)
 126    if match:
 127        return {
 128            'src_mode': match.group(1),
 129            'dst_mode': match.group(2),
 130            'src_sha1': match.group(3),
 131            'dst_sha1': match.group(4),
 132            'status': match.group(5),
 133            'status_score': match.group(6),
 134            'src': match.group(7),
 135            'dst': match.group(10)
 136        }
 137    return None
 138
 139def isModeExec(mode):
 140    # Returns True if the given git mode represents an executable file,
 141    # otherwise False.
 142    return mode[-3:] == "755"
 143
 144def isModeExecChanged(src_mode, dst_mode):
 145    return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148    cmd = "p4 -G %s" % cmd
 149    if verbose:
 150        sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152    # Use a temporary file to avoid deadlocks without
 153    # subprocess.communicate(), which would put another copy
 154    # of stdout into memory.
 155    stdin_file = None
 156    if stdin is not None:
 157        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158        stdin_file.write(stdin)
 159        stdin_file.flush()
 160        stdin_file.seek(0)
 161
 162    p4 = subprocess.Popen(cmd, shell=True,
 163                          stdin=stdin_file,
 164                          stdout=subprocess.PIPE)
 165
 166    result = []
 167    try:
 168        while True:
 169            entry = marshal.load(p4.stdout)
 170            result.append(entry)
 171    except EOFError:
 172        pass
 173    exitCode = p4.wait()
 174    if exitCode != 0:
 175        entry = {}
 176        entry["p4ExitCode"] = exitCode
 177        result.append(entry)
 178
 179    return result
 180
 181def p4Cmd(cmd):
 182    list = p4CmdList(cmd)
 183    result = {}
 184    for entry in list:
 185        result.update(entry)
 186    return result;
 187
 188def p4Where(depotPath):
 189    if not depotPath.endswith("/"):
 190        depotPath += "/"
 191    output = p4Cmd("where %s..." % depotPath)
 192    if output["code"] == "error":
 193        return ""
 194    clientPath = ""
 195    if "path" in output:
 196        clientPath = output.get("path")
 197    elif "data" in output:
 198        data = output.get("data")
 199        lastSpace = data.rfind(" ")
 200        clientPath = data[lastSpace + 1:]
 201
 202    if clientPath.endswith("..."):
 203        clientPath = clientPath[:-3]
 204    return clientPath
 205
 206def currentGitBranch():
 207    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209def isValidGitDir(path):
 210    if (os.path.exists(path + "/HEAD")
 211        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212        return True;
 213    return False
 214
 215def parseRevision(ref):
 216    return read_pipe("git rev-parse %s" % ref).strip()
 217
 218def extractLogMessageFromGitCommit(commit):
 219    logMessage = ""
 220
 221    ## fixme: title is first line of commit, not 1st paragraph.
 222    foundTitle = False
 223    for log in read_pipe_lines("git cat-file commit %s" % commit):
 224       if not foundTitle:
 225           if len(log) == 1:
 226               foundTitle = True
 227           continue
 228
 229       logMessage += log
 230    return logMessage
 231
 232def extractSettingsGitLog(log):
 233    values = {}
 234    for line in log.split("\n"):
 235        line = line.strip()
 236        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237        if not m:
 238            continue
 239
 240        assignments = m.group(1).split (':')
 241        for a in assignments:
 242            vals = a.split ('=')
 243            key = vals[0].strip()
 244            val = ('='.join (vals[1:])).strip()
 245            if val.endswith ('\"') and val.startswith('"'):
 246                val = val[1:-1]
 247
 248            values[key] = val
 249
 250    paths = values.get("depot-paths")
 251    if not paths:
 252        paths = values.get("depot-path")
 253    if paths:
 254        values['depot-paths'] = paths.split(',')
 255    return values
 256
 257def gitBranchExists(branch):
 258    proc = subprocess.Popen(["git", "rev-parse", branch],
 259                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260    return proc.wait() == 0;
 261
 262def gitConfig(key):
 263    return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265def p4BranchesInGit(branchesAreInRemotes = True):
 266    branches = {}
 267
 268    cmdline = "git rev-parse --symbolic "
 269    if branchesAreInRemotes:
 270        cmdline += " --remotes"
 271    else:
 272        cmdline += " --branches"
 273
 274    for line in read_pipe_lines(cmdline):
 275        line = line.strip()
 276
 277        ## only import to p4/
 278        if not line.startswith('p4/') or line == "p4/HEAD":
 279            continue
 280        branch = line
 281
 282        # strip off p4
 283        branch = re.sub ("^p4/", "", line)
 284
 285        branches[branch] = parseRevision(line)
 286    return branches
 287
 288def findUpstreamBranchPoint(head = "HEAD"):
 289    branches = p4BranchesInGit()
 290    # map from depot-path to branch name
 291    branchByDepotPath = {}
 292    for branch in branches.keys():
 293        tip = branches[branch]
 294        log = extractLogMessageFromGitCommit(tip)
 295        settings = extractSettingsGitLog(log)
 296        if settings.has_key("depot-paths"):
 297            paths = ",".join(settings["depot-paths"])
 298            branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300    settings = None
 301    parent = 0
 302    while parent < 65535:
 303        commit = head + "~%s" % parent
 304        log = extractLogMessageFromGitCommit(commit)
 305        settings = extractSettingsGitLog(log)
 306        if settings.has_key("depot-paths"):
 307            paths = ",".join(settings["depot-paths"])
 308            if branchByDepotPath.has_key(paths):
 309                return [branchByDepotPath[paths], settings]
 310
 311        parent = parent + 1
 312
 313    return ["", settings]
 314
 315def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316    if not silent:
 317        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318               % localRefPrefix)
 319
 320    originPrefix = "origin/p4/"
 321
 322    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323        line = line.strip()
 324        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325            continue
 326
 327        headName = line[len(originPrefix):]
 328        remoteHead = localRefPrefix + headName
 329        originHead = line
 330
 331        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332        if (not original.has_key('depot-paths')
 333            or not original.has_key('change')):
 334            continue
 335
 336        update = False
 337        if not gitBranchExists(remoteHead):
 338            if verbose:
 339                print "creating %s" % remoteHead
 340            update = True
 341        else:
 342            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343            if settings.has_key('change') > 0:
 344                if settings['depot-paths'] == original['depot-paths']:
 345                    originP4Change = int(original['change'])
 346                    p4Change = int(settings['change'])
 347                    if originP4Change > p4Change:
 348                        print ("%s (%s) is newer than %s (%s). "
 349                               "Updating p4 branch from origin."
 350                               % (originHead, originP4Change,
 351                                  remoteHead, p4Change))
 352                        update = True
 353                else:
 354                    print ("Ignoring: %s was imported from %s while "
 355                           "%s was imported from %s"
 356                           % (originHead, ','.join(original['depot-paths']),
 357                              remoteHead, ','.join(settings['depot-paths'])))
 358
 359        if update:
 360            system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362def originP4BranchesExist():
 363        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365def p4ChangesForPaths(depotPaths, changeRange):
 366    assert depotPaths
 367    output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                        for p in depotPaths]))
 369
 370    changes = []
 371    for line in output:
 372        changeNum = line.split(" ")[1]
 373        changes.append(int(changeNum))
 374
 375    changes.sort()
 376    return changes
 377
 378class Command:
 379    def __init__(self):
 380        self.usage = "usage: %prog [options]"
 381        self.needsGit = True
 382
 383class P4Debug(Command):
 384    def __init__(self):
 385        Command.__init__(self)
 386        self.options = [
 387            optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                 default=False),
 389            ]
 390        self.description = "A tool to debug the output of p4 -G."
 391        self.needsGit = False
 392        self.verbose = False
 393
 394    def run(self, args):
 395        j = 0
 396        for output in p4CmdList(" ".join(args)):
 397            print 'Element: %d' % j
 398            j += 1
 399            print output
 400        return True
 401
 402class P4RollBack(Command):
 403    def __init__(self):
 404        Command.__init__(self)
 405        self.options = [
 406            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408        ]
 409        self.description = "A tool to debug the multi-branch import. Don't use :)"
 410        self.verbose = False
 411        self.rollbackLocalBranches = False
 412
 413    def run(self, args):
 414        if len(args) != 1:
 415            return False
 416        maxChange = int(args[0])
 417
 418        if "p4ExitCode" in p4Cmd("changes -m 1"):
 419            die("Problems executing p4");
 420
 421        if self.rollbackLocalBranches:
 422            refPrefix = "refs/heads/"
 423            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424        else:
 425            refPrefix = "refs/remotes/"
 426            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428        for line in lines:
 429            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                line = line.strip()
 431                ref = refPrefix + line
 432                log = extractLogMessageFromGitCommit(ref)
 433                settings = extractSettingsGitLog(log)
 434
 435                depotPaths = settings['depot-paths']
 436                change = settings['change']
 437
 438                changed = False
 439
 440                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                           for p in depotPaths]))) == 0:
 442                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                    continue
 445
 446                while change and int(change) > maxChange:
 447                    changed = True
 448                    if self.verbose:
 449                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                    system("git update-ref %s \"%s^\"" % (ref, ref))
 451                    log = extractLogMessageFromGitCommit(ref)
 452                    settings =  extractSettingsGitLog(log)
 453
 454
 455                    depotPaths = settings['depot-paths']
 456                    change = settings['change']
 457
 458                if changed:
 459                    print "%s rewound to %s" % (ref, change)
 460
 461        return True
 462
 463class P4Submit(Command):
 464    def __init__(self):
 465        Command.__init__(self)
 466        self.options = [
 467                optparse.make_option("--continue", action="store_false", dest="firstTime"),
 468                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 469                optparse.make_option("--origin", dest="origin"),
 470                optparse.make_option("--reset", action="store_true", dest="reset"),
 471                optparse.make_option("--log-substitutions", dest="substFile"),
 472                optparse.make_option("--direct", dest="directSubmit", action="store_true"),
 473                optparse.make_option("-M", dest="detectRename", action="store_true"),
 474        ]
 475        self.description = "Submit changes from git to the perforce depot."
 476        self.usage += " [name of git branch to submit into perforce depot]"
 477        self.firstTime = True
 478        self.reset = False
 479        self.interactive = True
 480        self.substFile = ""
 481        self.firstTime = True
 482        self.origin = ""
 483        self.directSubmit = False
 484        self.detectRename = False
 485        self.verbose = False
 486        self.isWindows = (platform.system() == "Windows")
 487
 488        self.logSubstitutions = {}
 489        self.logSubstitutions["<enter description here>"] = "%log%"
 490        self.logSubstitutions["\tDetails:"] = "\tDetails:  %log%"
 491
 492    def check(self):
 493        if len(p4CmdList("opened ...")) > 0:
 494            die("You have files opened with perforce! Close them before starting the sync.")
 495
 496    def start(self):
 497        if len(self.config) > 0 and not self.reset:
 498            die("Cannot start sync. Previous sync config found at %s\n"
 499                "If you want to start submitting again from scratch "
 500                "maybe you want to call git-p4 submit --reset" % self.configFile)
 501
 502        commits = []
 503        if self.directSubmit:
 504            commits.append("0")
 505        else:
 506            for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 507                commits.append(line.strip())
 508            commits.reverse()
 509
 510        self.config["commits"] = commits
 511
 512    def prepareLogMessage(self, template, message):
 513        result = ""
 514
 515        for line in template.split("\n"):
 516            if line.startswith("#"):
 517                result += line + "\n"
 518                continue
 519
 520            substituted = False
 521            for key in self.logSubstitutions.keys():
 522                if line.find(key) != -1:
 523                    value = self.logSubstitutions[key]
 524                    value = value.replace("%log%", message)
 525                    if value != "@remove@":
 526                        result += line.replace(key, value) + "\n"
 527                    substituted = True
 528                    break
 529
 530            if not substituted:
 531                result += line + "\n"
 532
 533        return result
 534
 535    def prepareSubmitTemplate(self):
 536        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 537        template = ""
 538        inFilesSection = False
 539        for line in read_pipe_lines("p4 change -o"):
 540            if inFilesSection:
 541                if line.startswith("\t"):
 542                    # path starts and ends with a tab
 543                    path = line[1:]
 544                    lastTab = path.rfind("\t")
 545                    if lastTab != -1:
 546                        path = path[:lastTab]
 547                        if not path.startswith(self.depotPath):
 548                            continue
 549                else:
 550                    inFilesSection = False
 551            else:
 552                if line.startswith("Files:"):
 553                    inFilesSection = True
 554
 555            template += line
 556
 557        return template
 558
 559    def applyCommit(self, id):
 560        if self.directSubmit:
 561            print "Applying local change in working directory/index"
 562            diff = self.diffStatus
 563        else:
 564            print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 565            diffOpts = ("", "-M")[self.detectRename]
 566            diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 567        filesToAdd = set()
 568        filesToDelete = set()
 569        editedFiles = set()
 570        filesToChangeExecBit = {}
 571        for line in diff:
 572            diff = parseDiffTreeEntry(line)
 573            modifier = diff['status']
 574            path = diff['src']
 575            if modifier == "M":
 576                system("p4 edit \"%s\"" % path)
 577                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 578                    filesToChangeExecBit[path] = diff['dst_mode']
 579                editedFiles.add(path)
 580            elif modifier == "A":
 581                filesToAdd.add(path)
 582                filesToChangeExecBit[path] = diff['dst_mode']
 583                if path in filesToDelete:
 584                    filesToDelete.remove(path)
 585            elif modifier == "D":
 586                filesToDelete.add(path)
 587                if path in filesToAdd:
 588                    filesToAdd.remove(path)
 589            elif modifier == "R":
 590                src, dest = diff['src'], diff['dst']
 591                system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 592                system("p4 edit \"%s\"" % (dest))
 593                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 594                    filesToChangeExecBit[dest] = diff['dst_mode']
 595                os.unlink(dest)
 596                editedFiles.add(dest)
 597                filesToDelete.add(src)
 598            else:
 599                die("unknown modifier %s for %s" % (modifier, path))
 600
 601        if self.directSubmit:
 602            diffcmd = "cat \"%s\"" % self.diffFile
 603        else:
 604            diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 605        patchcmd = diffcmd + " | git apply "
 606        tryPatchCmd = patchcmd + "--check -"
 607        applyPatchCmd = patchcmd + "--check --apply -"
 608
 609        if os.system(tryPatchCmd) != 0:
 610            print "Unfortunately applying the change failed!"
 611            print "What do you want to do?"
 612            response = "x"
 613            while response != "s" and response != "a" and response != "w":
 614                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 615                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 616            if response == "s":
 617                print "Skipping! Good luck with the next patches..."
 618                for f in editedFiles:
 619                    system("p4 revert \"%s\"" % f);
 620                for f in filesToAdd:
 621                    system("rm %s" %f)
 622                return
 623            elif response == "a":
 624                os.system(applyPatchCmd)
 625                if len(filesToAdd) > 0:
 626                    print "You may also want to call p4 add on the following files:"
 627                    print " ".join(filesToAdd)
 628                if len(filesToDelete):
 629                    print "The following files should be scheduled for deletion with p4 delete:"
 630                    print " ".join(filesToDelete)
 631                die("Please resolve and submit the conflict manually and "
 632                    + "continue afterwards with git-p4 submit --continue")
 633            elif response == "w":
 634                system(diffcmd + " > patch.txt")
 635                print "Patch saved to patch.txt in %s !" % self.clientPath
 636                die("Please resolve and submit the conflict manually and "
 637                    "continue afterwards with git-p4 submit --continue")
 638
 639        system(applyPatchCmd)
 640
 641        for f in filesToAdd:
 642            system("p4 add \"%s\"" % f)
 643        for f in filesToDelete:
 644            system("p4 revert \"%s\"" % f)
 645            system("p4 delete \"%s\"" % f)
 646
 647        # Set/clear executable bits
 648        for f in filesToChangeExecBit.keys():
 649            mode = filesToChangeExecBit[f]
 650            setP4ExecBit(f, mode)
 651
 652        logMessage = ""
 653        if not self.directSubmit:
 654            logMessage = extractLogMessageFromGitCommit(id)
 655            logMessage = logMessage.replace("\n", "\n\t")
 656            if self.isWindows:
 657                logMessage = logMessage.replace("\n", "\r\n")
 658            logMessage = logMessage.strip()
 659
 660        template = self.prepareSubmitTemplate()
 661
 662        if self.interactive:
 663            submitTemplate = self.prepareLogMessage(template, logMessage)
 664            diff = read_pipe("p4 diff -du ...")
 665
 666            for newFile in filesToAdd:
 667                diff += "==== new file ====\n"
 668                diff += "--- /dev/null\n"
 669                diff += "+++ %s\n" % newFile
 670                f = open(newFile, "r")
 671                for line in f.readlines():
 672                    diff += "+" + line
 673                f.close()
 674
 675            separatorLine = "######## everything below this line is just the diff #######"
 676            if platform.system() == "Windows":
 677                separatorLine += "\r"
 678            separatorLine += "\n"
 679
 680            [handle, fileName] = tempfile.mkstemp()
 681            tmpFile = os.fdopen(handle, "w+")
 682            tmpFile.write(submitTemplate + separatorLine + diff)
 683            tmpFile.close()
 684            defaultEditor = "vi"
 685            if platform.system() == "Windows":
 686                defaultEditor = "notepad"
 687            editor = os.environ.get("EDITOR", defaultEditor);
 688            system(editor + " " + fileName)
 689            tmpFile = open(fileName, "rb")
 690            message = tmpFile.read()
 691            tmpFile.close()
 692            os.remove(fileName)
 693            submitTemplate = message[:message.index(separatorLine)]
 694            if self.isWindows:
 695                submitTemplate = submitTemplate.replace("\r\n", "\n")
 696
 697            if self.directSubmit:
 698                print "Submitting to git first"
 699                os.chdir(self.oldWorkingDirectory)
 700                write_pipe("git commit -a -F -", submitTemplate)
 701                os.chdir(self.clientPath)
 702
 703            write_pipe("p4 submit -i", submitTemplate)
 704        else:
 705            fileName = "submit.txt"
 706            file = open(fileName, "w+")
 707            file.write(self.prepareLogMessage(template, logMessage))
 708            file.close()
 709            print ("Perforce submit template written as %s. "
 710                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 711                   % (fileName, fileName))
 712
 713    def run(self, args):
 714        if len(args) == 0:
 715            self.master = currentGitBranch()
 716            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 717                die("Detecting current git branch failed!")
 718        elif len(args) == 1:
 719            self.master = args[0]
 720        else:
 721            return False
 722
 723        [upstream, settings] = findUpstreamBranchPoint()
 724        self.depotPath = settings['depot-paths'][0]
 725        if len(self.origin) == 0:
 726            self.origin = upstream
 727
 728        if self.verbose:
 729            print "Origin branch is " + self.origin
 730
 731        if len(self.depotPath) == 0:
 732            print "Internal error: cannot locate perforce depot path from existing branches"
 733            sys.exit(128)
 734
 735        self.clientPath = p4Where(self.depotPath)
 736
 737        if len(self.clientPath) == 0:
 738            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 739            sys.exit(128)
 740
 741        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 742        self.oldWorkingDirectory = os.getcwd()
 743
 744        if self.directSubmit:
 745            self.diffStatus = read_pipe_lines("git diff -r --name-status HEAD")
 746            if len(self.diffStatus) == 0:
 747                print "No changes in working directory to submit."
 748                return True
 749            patch = read_pipe("git diff -p --binary --diff-filter=ACMRTUXB HEAD")
 750            self.diffFile = self.gitdir + "/p4-git-diff"
 751            f = open(self.diffFile, "wb")
 752            f.write(patch)
 753            f.close();
 754
 755        os.chdir(self.clientPath)
 756        print "Syncronizing p4 checkout..."
 757        system("p4 sync ...")
 758
 759        if self.reset:
 760            self.firstTime = True
 761
 762        if len(self.substFile) > 0:
 763            for line in open(self.substFile, "r").readlines():
 764                tokens = line.strip().split("=")
 765                self.logSubstitutions[tokens[0]] = tokens[1]
 766
 767        self.check()
 768        self.configFile = self.gitdir + "/p4-git-sync.cfg"
 769        self.config = shelve.open(self.configFile, writeback=True)
 770
 771        if self.firstTime:
 772            self.start()
 773
 774        commits = self.config.get("commits", [])
 775
 776        while len(commits) > 0:
 777            self.firstTime = False
 778            commit = commits[0]
 779            commits = commits[1:]
 780            self.config["commits"] = commits
 781            self.applyCommit(commit)
 782            if not self.interactive:
 783                break
 784
 785        self.config.close()
 786
 787        if self.directSubmit:
 788            os.remove(self.diffFile)
 789
 790        if len(commits) == 0:
 791            if self.firstTime:
 792                print "No changes found to apply between %s and current HEAD" % self.origin
 793            else:
 794                print "All changes applied!"
 795                os.chdir(self.oldWorkingDirectory)
 796
 797                sync = P4Sync()
 798                sync.run([])
 799
 800                rebase = P4Rebase()
 801                rebase.rebase()
 802            os.remove(self.configFile)
 803
 804        return True
 805
 806class P4Sync(Command):
 807    def __init__(self):
 808        Command.__init__(self)
 809        self.options = [
 810                optparse.make_option("--branch", dest="branch"),
 811                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 812                optparse.make_option("--changesfile", dest="changesFile"),
 813                optparse.make_option("--silent", dest="silent", action="store_true"),
 814                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 815                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 816                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 817                                     help="Import into refs/heads/ , not refs/remotes"),
 818                optparse.make_option("--max-changes", dest="maxChanges"),
 819                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 820                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import")
 821        ]
 822        self.description = """Imports from Perforce into a git repository.\n
 823    example:
 824    //depot/my/project/ -- to import the current head
 825    //depot/my/project/@all -- to import everything
 826    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 827
 828    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 829
 830        self.usage += " //depot/path[@revRange]"
 831        self.silent = False
 832        self.createdBranches = Set()
 833        self.committedChanges = Set()
 834        self.branch = ""
 835        self.detectBranches = False
 836        self.detectLabels = False
 837        self.changesFile = ""
 838        self.syncWithOrigin = True
 839        self.verbose = False
 840        self.importIntoRemotes = True
 841        self.maxChanges = ""
 842        self.isWindows = (platform.system() == "Windows")
 843        self.keepRepoPath = False
 844        self.depotPaths = None
 845        self.p4BranchesInGit = []
 846        self.cloneExclude = []
 847
 848        if gitConfig("git-p4.syncFromOrigin") == "false":
 849            self.syncWithOrigin = False
 850
 851    def extractFilesFromCommit(self, commit):
 852        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 853                             for path in self.cloneExclude]
 854        files = []
 855        fnum = 0
 856        while commit.has_key("depotFile%s" % fnum):
 857            path =  commit["depotFile%s" % fnum]
 858
 859            if [p for p in self.cloneExclude
 860                if path.startswith (p)]:
 861                found = False
 862            else:
 863                found = [p for p in self.depotPaths
 864                         if path.startswith (p)]
 865            if not found:
 866                fnum = fnum + 1
 867                continue
 868
 869            file = {}
 870            file["path"] = path
 871            file["rev"] = commit["rev%s" % fnum]
 872            file["action"] = commit["action%s" % fnum]
 873            file["type"] = commit["type%s" % fnum]
 874            files.append(file)
 875            fnum = fnum + 1
 876        return files
 877
 878    def stripRepoPath(self, path, prefixes):
 879        if self.keepRepoPath:
 880            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 881
 882        for p in prefixes:
 883            if path.startswith(p):
 884                path = path[len(p):]
 885
 886        return path
 887
 888    def splitFilesIntoBranches(self, commit):
 889        branches = {}
 890        fnum = 0
 891        while commit.has_key("depotFile%s" % fnum):
 892            path =  commit["depotFile%s" % fnum]
 893            found = [p for p in self.depotPaths
 894                     if path.startswith (p)]
 895            if not found:
 896                fnum = fnum + 1
 897                continue
 898
 899            file = {}
 900            file["path"] = path
 901            file["rev"] = commit["rev%s" % fnum]
 902            file["action"] = commit["action%s" % fnum]
 903            file["type"] = commit["type%s" % fnum]
 904            fnum = fnum + 1
 905
 906            relPath = self.stripRepoPath(path, self.depotPaths)
 907
 908            for branch in self.knownBranches.keys():
 909
 910                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 911                if relPath.startswith(branch + "/"):
 912                    if branch not in branches:
 913                        branches[branch] = []
 914                    branches[branch].append(file)
 915                    break
 916
 917        return branches
 918
 919    ## Should move this out, doesn't use SELF.
 920    def readP4Files(self, files):
 921        files = [f for f in files
 922                 if f['action'] != 'delete']
 923
 924        if not files:
 925            return
 926
 927        filedata = p4CmdList('-x - print',
 928                             stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 929                                              for f in files]),
 930                             stdin_mode='w+')
 931        if "p4ExitCode" in filedata[0]:
 932            die("Problems executing p4. Error: [%d]."
 933                % (filedata[0]['p4ExitCode']));
 934
 935        j = 0;
 936        contents = {}
 937        while j < len(filedata):
 938            stat = filedata[j]
 939            j += 1
 940            text = ''
 941            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 942                tmp = filedata[j]['data']
 943                if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 944                    tmp = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', tmp)
 945                elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 946                    tmp = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', tmp)
 947                text += tmp
 948                j += 1
 949
 950
 951            if not stat.has_key('depotFile'):
 952                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 953                continue
 954
 955            contents[stat['depotFile']] = text
 956
 957        for f in files:
 958            assert not f.has_key('data')
 959            f['data'] = contents[f['path']]
 960
 961    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 962        epoch = details["time"]
 963        author = details["user"]
 964
 965        if self.verbose:
 966            print "commit into %s" % branch
 967
 968        # start with reading files; if that fails, we should not
 969        # create a commit.
 970        new_files = []
 971        for f in files:
 972            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 973                new_files.append (f)
 974            else:
 975                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 976        files = new_files
 977        self.readP4Files(files)
 978
 979
 980
 981
 982        self.gitStream.write("commit %s\n" % branch)
 983#        gitStream.write("mark :%s\n" % details["change"])
 984        self.committedChanges.add(int(details["change"]))
 985        committer = ""
 986        if author not in self.users:
 987            self.getUserMapFromPerforceServer()
 988        if author in self.users:
 989            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 990        else:
 991            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 992
 993        self.gitStream.write("committer %s\n" % committer)
 994
 995        self.gitStream.write("data <<EOT\n")
 996        self.gitStream.write(details["desc"])
 997        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 998                             % (','.join (branchPrefixes), details["change"]))
 999        if len(details['options']) > 0:
1000            self.gitStream.write(": options = %s" % details['options'])
1001        self.gitStream.write("]\nEOT\n\n")
1002
1003        if len(parent) > 0:
1004            if self.verbose:
1005                print "parent %s" % parent
1006            self.gitStream.write("from %s\n" % parent)
1007
1008        for file in files:
1009            if file["type"] == "apple":
1010                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1011                continue
1012
1013            relPath = self.stripRepoPath(file['path'], branchPrefixes)
1014            if file["action"] == "delete":
1015                self.gitStream.write("D %s\n" % relPath)
1016            else:
1017                data = file['data']
1018
1019                mode = "644"
1020                if isP4Exec(file["type"]):
1021                    mode = "755"
1022                elif file["type"] == "symlink":
1023                    mode = "120000"
1024                    # p4 print on a symlink contains "target\n", so strip it off
1025                    data = data[:-1]
1026
1027                if self.isWindows and file["type"].endswith("text"):
1028                    data = data.replace("\r\n", "\n")
1029
1030                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1031                self.gitStream.write("data %s\n" % len(data))
1032                self.gitStream.write(data)
1033                self.gitStream.write("\n")
1034
1035        self.gitStream.write("\n")
1036
1037        change = int(details["change"])
1038
1039        if self.labels.has_key(change):
1040            label = self.labels[change]
1041            labelDetails = label[0]
1042            labelRevisions = label[1]
1043            if self.verbose:
1044                print "Change %s is labelled %s" % (change, labelDetails)
1045
1046            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1047                                                    for p in branchPrefixes]))
1048
1049            if len(files) == len(labelRevisions):
1050
1051                cleanedFiles = {}
1052                for info in files:
1053                    if info["action"] == "delete":
1054                        continue
1055                    cleanedFiles[info["depotFile"]] = info["rev"]
1056
1057                if cleanedFiles == labelRevisions:
1058                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1059                    self.gitStream.write("from %s\n" % branch)
1060
1061                    owner = labelDetails["Owner"]
1062                    tagger = ""
1063                    if author in self.users:
1064                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1065                    else:
1066                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1067                    self.gitStream.write("tagger %s\n" % tagger)
1068                    self.gitStream.write("data <<EOT\n")
1069                    self.gitStream.write(labelDetails["Description"])
1070                    self.gitStream.write("EOT\n\n")
1071
1072                else:
1073                    if not self.silent:
1074                        print ("Tag %s does not match with change %s: files do not match."
1075                               % (labelDetails["label"], change))
1076
1077            else:
1078                if not self.silent:
1079                    print ("Tag %s does not match with change %s: file count is different."
1080                           % (labelDetails["label"], change))
1081
1082    def getUserCacheFilename(self):
1083        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1084        return home + "/.gitp4-usercache.txt"
1085
1086    def getUserMapFromPerforceServer(self):
1087        if self.userMapFromPerforceServer:
1088            return
1089        self.users = {}
1090
1091        for output in p4CmdList("users"):
1092            if not output.has_key("User"):
1093                continue
1094            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1095
1096
1097        s = ''
1098        for (key, val) in self.users.items():
1099            s += "%s\t%s\n" % (key, val)
1100
1101        open(self.getUserCacheFilename(), "wb").write(s)
1102        self.userMapFromPerforceServer = True
1103
1104    def loadUserMapFromCache(self):
1105        self.users = {}
1106        self.userMapFromPerforceServer = False
1107        try:
1108            cache = open(self.getUserCacheFilename(), "rb")
1109            lines = cache.readlines()
1110            cache.close()
1111            for line in lines:
1112                entry = line.strip().split("\t")
1113                self.users[entry[0]] = entry[1]
1114        except IOError:
1115            self.getUserMapFromPerforceServer()
1116
1117    def getLabels(self):
1118        self.labels = {}
1119
1120        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1121        if len(l) > 0 and not self.silent:
1122            print "Finding files belonging to labels in %s" % `self.depotPaths`
1123
1124        for output in l:
1125            label = output["label"]
1126            revisions = {}
1127            newestChange = 0
1128            if self.verbose:
1129                print "Querying files for label %s" % label
1130            for file in p4CmdList("files "
1131                                  +  ' '.join (["%s...@%s" % (p, label)
1132                                                for p in self.depotPaths])):
1133                revisions[file["depotFile"]] = file["rev"]
1134                change = int(file["change"])
1135                if change > newestChange:
1136                    newestChange = change
1137
1138            self.labels[newestChange] = [output, revisions]
1139
1140        if self.verbose:
1141            print "Label changes: %s" % self.labels.keys()
1142
1143    def guessProjectName(self):
1144        for p in self.depotPaths:
1145            if p.endswith("/"):
1146                p = p[:-1]
1147            p = p[p.strip().rfind("/") + 1:]
1148            if not p.endswith("/"):
1149               p += "/"
1150            return p
1151
1152    def getBranchMapping(self):
1153        lostAndFoundBranches = set()
1154
1155        for info in p4CmdList("branches"):
1156            details = p4Cmd("branch -o %s" % info["branch"])
1157            viewIdx = 0
1158            while details.has_key("View%s" % viewIdx):
1159                paths = details["View%s" % viewIdx].split(" ")
1160                viewIdx = viewIdx + 1
1161                # require standard //depot/foo/... //depot/bar/... mapping
1162                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1163                    continue
1164                source = paths[0]
1165                destination = paths[1]
1166                ## HACK
1167                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1168                    source = source[len(self.depotPaths[0]):-4]
1169                    destination = destination[len(self.depotPaths[0]):-4]
1170
1171                    if destination in self.knownBranches:
1172                        if not self.silent:
1173                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1174                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1175                        continue
1176
1177                    self.knownBranches[destination] = source
1178
1179                    lostAndFoundBranches.discard(destination)
1180
1181                    if source not in self.knownBranches:
1182                        lostAndFoundBranches.add(source)
1183
1184
1185        for branch in lostAndFoundBranches:
1186            self.knownBranches[branch] = branch
1187
1188    def getBranchMappingFromGitBranches(self):
1189        branches = p4BranchesInGit(self.importIntoRemotes)
1190        for branch in branches.keys():
1191            if branch == "master":
1192                branch = "main"
1193            else:
1194                branch = branch[len(self.projectName):]
1195            self.knownBranches[branch] = branch
1196
1197    def listExistingP4GitBranches(self):
1198        # branches holds mapping from name to commit
1199        branches = p4BranchesInGit(self.importIntoRemotes)
1200        self.p4BranchesInGit = branches.keys()
1201        for branch in branches.keys():
1202            self.initialParents[self.refPrefix + branch] = branches[branch]
1203
1204    def updateOptionDict(self, d):
1205        option_keys = {}
1206        if self.keepRepoPath:
1207            option_keys['keepRepoPath'] = 1
1208
1209        d["options"] = ' '.join(sorted(option_keys.keys()))
1210
1211    def readOptions(self, d):
1212        self.keepRepoPath = (d.has_key('options')
1213                             and ('keepRepoPath' in d['options']))
1214
1215    def gitRefForBranch(self, branch):
1216        if branch == "main":
1217            return self.refPrefix + "master"
1218
1219        if len(branch) <= 0:
1220            return branch
1221
1222        return self.refPrefix + self.projectName + branch
1223
1224    def gitCommitByP4Change(self, ref, change):
1225        if self.verbose:
1226            print "looking in ref " + ref + " for change %s using bisect..." % change
1227
1228        earliestCommit = ""
1229        latestCommit = parseRevision(ref)
1230
1231        while True:
1232            if self.verbose:
1233                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1234            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1235            if len(next) == 0:
1236                if self.verbose:
1237                    print "argh"
1238                return ""
1239            log = extractLogMessageFromGitCommit(next)
1240            settings = extractSettingsGitLog(log)
1241            currentChange = int(settings['change'])
1242            if self.verbose:
1243                print "current change %s" % currentChange
1244
1245            if currentChange == change:
1246                if self.verbose:
1247                    print "found %s" % next
1248                return next
1249
1250            if currentChange < change:
1251                earliestCommit = "^%s" % next
1252            else:
1253                latestCommit = "%s" % next
1254
1255        return ""
1256
1257    def importNewBranch(self, branch, maxChange):
1258        # make fast-import flush all changes to disk and update the refs using the checkpoint
1259        # command so that we can try to find the branch parent in the git history
1260        self.gitStream.write("checkpoint\n\n");
1261        self.gitStream.flush();
1262        branchPrefix = self.depotPaths[0] + branch + "/"
1263        range = "@1,%s" % maxChange
1264        #print "prefix" + branchPrefix
1265        changes = p4ChangesForPaths([branchPrefix], range)
1266        if len(changes) <= 0:
1267            return False
1268        firstChange = changes[0]
1269        #print "first change in branch: %s" % firstChange
1270        sourceBranch = self.knownBranches[branch]
1271        sourceDepotPath = self.depotPaths[0] + sourceBranch
1272        sourceRef = self.gitRefForBranch(sourceBranch)
1273        #print "source " + sourceBranch
1274
1275        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1276        #print "branch parent: %s" % branchParentChange
1277        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1278        if len(gitParent) > 0:
1279            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1280            #print "parent git commit: %s" % gitParent
1281
1282        self.importChanges(changes)
1283        return True
1284
1285    def importChanges(self, changes):
1286        cnt = 1
1287        for change in changes:
1288            description = p4Cmd("describe %s" % change)
1289            self.updateOptionDict(description)
1290
1291            if not self.silent:
1292                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1293                sys.stdout.flush()
1294            cnt = cnt + 1
1295
1296            try:
1297                if self.detectBranches:
1298                    branches = self.splitFilesIntoBranches(description)
1299                    for branch in branches.keys():
1300                        ## HACK  --hwn
1301                        branchPrefix = self.depotPaths[0] + branch + "/"
1302
1303                        parent = ""
1304
1305                        filesForCommit = branches[branch]
1306
1307                        if self.verbose:
1308                            print "branch is %s" % branch
1309
1310                        self.updatedBranches.add(branch)
1311
1312                        if branch not in self.createdBranches:
1313                            self.createdBranches.add(branch)
1314                            parent = self.knownBranches[branch]
1315                            if parent == branch:
1316                                parent = ""
1317                            else:
1318                                fullBranch = self.projectName + branch
1319                                if fullBranch not in self.p4BranchesInGit:
1320                                    if not self.silent:
1321                                        print("\n    Importing new branch %s" % fullBranch);
1322                                    if self.importNewBranch(branch, change - 1):
1323                                        parent = ""
1324                                        self.p4BranchesInGit.append(fullBranch)
1325                                    if not self.silent:
1326                                        print("\n    Resuming with change %s" % change);
1327
1328                                if self.verbose:
1329                                    print "parent determined through known branches: %s" % parent
1330
1331                        branch = self.gitRefForBranch(branch)
1332                        parent = self.gitRefForBranch(parent)
1333
1334                        if self.verbose:
1335                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1336
1337                        if len(parent) == 0 and branch in self.initialParents:
1338                            parent = self.initialParents[branch]
1339                            del self.initialParents[branch]
1340
1341                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1342                else:
1343                    files = self.extractFilesFromCommit(description)
1344                    self.commit(description, files, self.branch, self.depotPaths,
1345                                self.initialParent)
1346                    self.initialParent = ""
1347            except IOError:
1348                print self.gitError.read()
1349                sys.exit(1)
1350
1351    def importHeadRevision(self, revision):
1352        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1353
1354        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1355        details["desc"] = ("Initial import of %s from the state at revision %s"
1356                           % (' '.join(self.depotPaths), revision))
1357        details["change"] = revision
1358        newestRevision = 0
1359
1360        fileCnt = 0
1361        for info in p4CmdList("files "
1362                              +  ' '.join(["%s...%s"
1363                                           % (p, revision)
1364                                           for p in self.depotPaths])):
1365
1366            if info['code'] == 'error':
1367                sys.stderr.write("p4 returned an error: %s\n"
1368                                 % info['data'])
1369                sys.exit(1)
1370
1371
1372            change = int(info["change"])
1373            if change > newestRevision:
1374                newestRevision = change
1375
1376            if info["action"] == "delete":
1377                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1378                #fileCnt = fileCnt + 1
1379                continue
1380
1381            for prop in ["depotFile", "rev", "action", "type" ]:
1382                details["%s%s" % (prop, fileCnt)] = info[prop]
1383
1384            fileCnt = fileCnt + 1
1385
1386        details["change"] = newestRevision
1387        self.updateOptionDict(details)
1388        try:
1389            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1390        except IOError:
1391            print "IO error with git fast-import. Is your git version recent enough?"
1392            print self.gitError.read()
1393
1394
1395    def run(self, args):
1396        self.depotPaths = []
1397        self.changeRange = ""
1398        self.initialParent = ""
1399        self.previousDepotPaths = []
1400
1401        # map from branch depot path to parent branch
1402        self.knownBranches = {}
1403        self.initialParents = {}
1404        self.hasOrigin = originP4BranchesExist()
1405        if not self.syncWithOrigin:
1406            self.hasOrigin = False
1407
1408        if self.importIntoRemotes:
1409            self.refPrefix = "refs/remotes/p4/"
1410        else:
1411            self.refPrefix = "refs/heads/p4/"
1412
1413        if self.syncWithOrigin and self.hasOrigin:
1414            if not self.silent:
1415                print "Syncing with origin first by calling git fetch origin"
1416            system("git fetch origin")
1417
1418        if len(self.branch) == 0:
1419            self.branch = self.refPrefix + "master"
1420            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1421                system("git update-ref %s refs/heads/p4" % self.branch)
1422                system("git branch -D p4");
1423            # create it /after/ importing, when master exists
1424            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1425                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1426
1427        # TODO: should always look at previous commits,
1428        # merge with previous imports, if possible.
1429        if args == []:
1430            if self.hasOrigin:
1431                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1432            self.listExistingP4GitBranches()
1433
1434            if len(self.p4BranchesInGit) > 1:
1435                if not self.silent:
1436                    print "Importing from/into multiple branches"
1437                self.detectBranches = True
1438
1439            if self.verbose:
1440                print "branches: %s" % self.p4BranchesInGit
1441
1442            p4Change = 0
1443            for branch in self.p4BranchesInGit:
1444                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1445
1446                settings = extractSettingsGitLog(logMsg)
1447
1448                self.readOptions(settings)
1449                if (settings.has_key('depot-paths')
1450                    and settings.has_key ('change')):
1451                    change = int(settings['change']) + 1
1452                    p4Change = max(p4Change, change)
1453
1454                    depotPaths = sorted(settings['depot-paths'])
1455                    if self.previousDepotPaths == []:
1456                        self.previousDepotPaths = depotPaths
1457                    else:
1458                        paths = []
1459                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1460                            for i in range(0, min(len(cur), len(prev))):
1461                                if cur[i] <> prev[i]:
1462                                    i = i - 1
1463                                    break
1464
1465                            paths.append (cur[:i + 1])
1466
1467                        self.previousDepotPaths = paths
1468
1469            if p4Change > 0:
1470                self.depotPaths = sorted(self.previousDepotPaths)
1471                self.changeRange = "@%s,#head" % p4Change
1472                if not self.detectBranches:
1473                    self.initialParent = parseRevision(self.branch)
1474                if not self.silent and not self.detectBranches:
1475                    print "Performing incremental import into %s git branch" % self.branch
1476
1477        if not self.branch.startswith("refs/"):
1478            self.branch = "refs/heads/" + self.branch
1479
1480        if len(args) == 0 and self.depotPaths:
1481            if not self.silent:
1482                print "Depot paths: %s" % ' '.join(self.depotPaths)
1483        else:
1484            if self.depotPaths and self.depotPaths != args:
1485                print ("previous import used depot path %s and now %s was specified. "
1486                       "This doesn't work!" % (' '.join (self.depotPaths),
1487                                               ' '.join (args)))
1488                sys.exit(1)
1489
1490            self.depotPaths = sorted(args)
1491
1492        revision = ""
1493        self.users = {}
1494
1495        newPaths = []
1496        for p in self.depotPaths:
1497            if p.find("@") != -1:
1498                atIdx = p.index("@")
1499                self.changeRange = p[atIdx:]
1500                if self.changeRange == "@all":
1501                    self.changeRange = ""
1502                elif ',' not in self.changeRange:
1503                    revision = self.changeRange
1504                    self.changeRange = ""
1505                p = p[:atIdx]
1506            elif p.find("#") != -1:
1507                hashIdx = p.index("#")
1508                revision = p[hashIdx:]
1509                p = p[:hashIdx]
1510            elif self.previousDepotPaths == []:
1511                revision = "#head"
1512
1513            p = re.sub ("\.\.\.$", "", p)
1514            if not p.endswith("/"):
1515                p += "/"
1516
1517            newPaths.append(p)
1518
1519        self.depotPaths = newPaths
1520
1521
1522        self.loadUserMapFromCache()
1523        self.labels = {}
1524        if self.detectLabels:
1525            self.getLabels();
1526
1527        if self.detectBranches:
1528            ## FIXME - what's a P4 projectName ?
1529            self.projectName = self.guessProjectName()
1530
1531            if self.hasOrigin:
1532                self.getBranchMappingFromGitBranches()
1533            else:
1534                self.getBranchMapping()
1535            if self.verbose:
1536                print "p4-git branches: %s" % self.p4BranchesInGit
1537                print "initial parents: %s" % self.initialParents
1538            for b in self.p4BranchesInGit:
1539                if b != "master":
1540
1541                    ## FIXME
1542                    b = b[len(self.projectName):]
1543                self.createdBranches.add(b)
1544
1545        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1546
1547        importProcess = subprocess.Popen(["git", "fast-import"],
1548                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1549                                         stderr=subprocess.PIPE);
1550        self.gitOutput = importProcess.stdout
1551        self.gitStream = importProcess.stdin
1552        self.gitError = importProcess.stderr
1553
1554        if revision:
1555            self.importHeadRevision(revision)
1556        else:
1557            changes = []
1558
1559            if len(self.changesFile) > 0:
1560                output = open(self.changesFile).readlines()
1561                changeSet = Set()
1562                for line in output:
1563                    changeSet.add(int(line))
1564
1565                for change in changeSet:
1566                    changes.append(change)
1567
1568                changes.sort()
1569            else:
1570                if self.verbose:
1571                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1572                                                              self.changeRange)
1573                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1574
1575                if len(self.maxChanges) > 0:
1576                    changes = changes[:min(int(self.maxChanges), len(changes))]
1577
1578            if len(changes) == 0:
1579                if not self.silent:
1580                    print "No changes to import!"
1581                return True
1582
1583            if not self.silent and not self.detectBranches:
1584                print "Import destination: %s" % self.branch
1585
1586            self.updatedBranches = set()
1587
1588            self.importChanges(changes)
1589
1590            if not self.silent:
1591                print ""
1592                if len(self.updatedBranches) > 0:
1593                    sys.stdout.write("Updated branches: ")
1594                    for b in self.updatedBranches:
1595                        sys.stdout.write("%s " % b)
1596                    sys.stdout.write("\n")
1597
1598        self.gitStream.close()
1599        if importProcess.wait() != 0:
1600            die("fast-import failed: %s" % self.gitError.read())
1601        self.gitOutput.close()
1602        self.gitError.close()
1603
1604        return True
1605
1606class P4Rebase(Command):
1607    def __init__(self):
1608        Command.__init__(self)
1609        self.options = [ ]
1610        self.description = ("Fetches the latest revision from perforce and "
1611                            + "rebases the current work (branch) against it")
1612        self.verbose = False
1613
1614    def run(self, args):
1615        sync = P4Sync()
1616        sync.run([])
1617
1618        return self.rebase()
1619
1620    def rebase(self):
1621        if os.system("git update-index --refresh") != 0:
1622            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1623        if len(read_pipe("git diff-index HEAD --")) > 0:
1624            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1625
1626        [upstream, settings] = findUpstreamBranchPoint()
1627        if len(upstream) == 0:
1628            die("Cannot find upstream branchpoint for rebase")
1629
1630        # the branchpoint may be p4/foo~3, so strip off the parent
1631        upstream = re.sub("~[0-9]+$", "", upstream)
1632
1633        print "Rebasing the current branch onto %s" % upstream
1634        oldHead = read_pipe("git rev-parse HEAD").strip()
1635        system("git rebase %s" % upstream)
1636        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1637        return True
1638
1639class P4Clone(P4Sync):
1640    def __init__(self):
1641        P4Sync.__init__(self)
1642        self.description = "Creates a new git repository and imports from Perforce into it"
1643        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1644        self.options += [
1645            optparse.make_option("--destination", dest="cloneDestination",
1646                                 action='store', default=None,
1647                                 help="where to leave result of the clone"),
1648            optparse.make_option("-/", dest="cloneExclude",
1649                                 action="append", type="string",
1650                                 help="exclude depot path")
1651        ]
1652        self.cloneDestination = None
1653        self.needsGit = False
1654
1655    # This is required for the "append" cloneExclude action
1656    def ensure_value(self, attr, value):
1657        if not hasattr(self, attr) or getattr(self, attr) is None:
1658            setattr(self, attr, value)
1659        return getattr(self, attr)
1660
1661    def defaultDestination(self, args):
1662        ## TODO: use common prefix of args?
1663        depotPath = args[0]
1664        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1665        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1666        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1667        depotDir = re.sub(r"/$", "", depotDir)
1668        return os.path.split(depotDir)[1]
1669
1670    def run(self, args):
1671        if len(args) < 1:
1672            return False
1673
1674        if self.keepRepoPath and not self.cloneDestination:
1675            sys.stderr.write("Must specify destination for --keep-path\n")
1676            sys.exit(1)
1677
1678        depotPaths = args
1679
1680        if not self.cloneDestination and len(depotPaths) > 1:
1681            self.cloneDestination = depotPaths[-1]
1682            depotPaths = depotPaths[:-1]
1683
1684        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1685        for p in depotPaths:
1686            if not p.startswith("//"):
1687                return False
1688
1689        if not self.cloneDestination:
1690            self.cloneDestination = self.defaultDestination(args)
1691
1692        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1693        if not os.path.exists(self.cloneDestination):
1694            os.makedirs(self.cloneDestination)
1695        os.chdir(self.cloneDestination)
1696        system("git init")
1697        self.gitdir = os.getcwd() + "/.git"
1698        if not P4Sync.run(self, depotPaths):
1699            return False
1700        if self.branch != "master":
1701            if gitBranchExists("refs/remotes/p4/master"):
1702                system("git branch master refs/remotes/p4/master")
1703                system("git checkout -f")
1704            else:
1705                print "Could not detect main branch. No checkout/master branch created."
1706
1707        return True
1708
1709class P4Branches(Command):
1710    def __init__(self):
1711        Command.__init__(self)
1712        self.options = [ ]
1713        self.description = ("Shows the git branches that hold imports and their "
1714                            + "corresponding perforce depot paths")
1715        self.verbose = False
1716
1717    def run(self, args):
1718        if originP4BranchesExist():
1719            createOrUpdateBranchesFromOrigin()
1720
1721        cmdline = "git rev-parse --symbolic "
1722        cmdline += " --remotes"
1723
1724        for line in read_pipe_lines(cmdline):
1725            line = line.strip()
1726
1727            if not line.startswith('p4/') or line == "p4/HEAD":
1728                continue
1729            branch = line
1730
1731            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1732            settings = extractSettingsGitLog(log)
1733
1734            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1735        return True
1736
1737class HelpFormatter(optparse.IndentedHelpFormatter):
1738    def __init__(self):
1739        optparse.IndentedHelpFormatter.__init__(self)
1740
1741    def format_description(self, description):
1742        if description:
1743            return description + "\n"
1744        else:
1745            return ""
1746
1747def printUsage(commands):
1748    print "usage: %s <command> [options]" % sys.argv[0]
1749    print ""
1750    print "valid commands: %s" % ", ".join(commands)
1751    print ""
1752    print "Try %s <command> --help for command specific help." % sys.argv[0]
1753    print ""
1754
1755commands = {
1756    "debug" : P4Debug,
1757    "submit" : P4Submit,
1758    "commit" : P4Submit,
1759    "sync" : P4Sync,
1760    "rebase" : P4Rebase,
1761    "clone" : P4Clone,
1762    "rollback" : P4RollBack,
1763    "branches" : P4Branches
1764}
1765
1766
1767def main():
1768    if len(sys.argv[1:]) == 0:
1769        printUsage(commands.keys())
1770        sys.exit(2)
1771
1772    cmd = ""
1773    cmdName = sys.argv[1]
1774    try:
1775        klass = commands[cmdName]
1776        cmd = klass()
1777    except KeyError:
1778        print "unknown command %s" % cmdName
1779        print ""
1780        printUsage(commands.keys())
1781        sys.exit(2)
1782
1783    options = cmd.options
1784    cmd.gitdir = os.environ.get("GIT_DIR", None)
1785
1786    args = sys.argv[2:]
1787
1788    if len(options) > 0:
1789        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1790
1791        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1792                                       options,
1793                                       description = cmd.description,
1794                                       formatter = HelpFormatter())
1795
1796        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1797    global verbose
1798    verbose = cmd.verbose
1799    if cmd.needsGit:
1800        if cmd.gitdir == None:
1801            cmd.gitdir = os.path.abspath(".git")
1802            if not isValidGitDir(cmd.gitdir):
1803                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1804                if os.path.exists(cmd.gitdir):
1805                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1806                    if len(cdup) > 0:
1807                        os.chdir(cdup);
1808
1809        if not isValidGitDir(cmd.gitdir):
1810            if isValidGitDir(cmd.gitdir + "/.git"):
1811                cmd.gitdir += "/.git"
1812            else:
1813                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1814
1815        os.environ["GIT_DIR"] = cmd.gitdir
1816
1817    if not cmd.run(args):
1818        parser.print_help()
1819
1820
1821if __name__ == '__main__':
1822    main()