Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit git-p4: Removed git-p4 submit --direct. (0e36f2d)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19def die(msg):
  20    if verbose:
  21        raise Exception(msg)
  22    else:
  23        sys.stderr.write(msg + "\n")
  24        sys.exit(1)
  25
  26def write_pipe(c, str):
  27    if verbose:
  28        sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30    pipe = os.popen(c, 'w')
  31    val = pipe.write(str)
  32    if pipe.close():
  33        die('Command failed: %s' % c)
  34
  35    return val
  36
  37def read_pipe(c, ignore_error=False):
  38    if verbose:
  39        sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41    pipe = os.popen(c, 'rb')
  42    val = pipe.read()
  43    if pipe.close() and not ignore_error:
  44        die('Command failed: %s' % c)
  45
  46    return val
  47
  48
  49def read_pipe_lines(c):
  50    if verbose:
  51        sys.stderr.write('Reading pipe: %s\n' % c)
  52    ## todo: check return status
  53    pipe = os.popen(c, 'rb')
  54    val = pipe.readlines()
  55    if pipe.close():
  56        die('Command failed: %s' % c)
  57
  58    return val
  59
  60def system(cmd):
  61    if verbose:
  62        sys.stderr.write("executing %s\n" % cmd)
  63    if os.system(cmd) != 0:
  64        die("command failed: %s" % cmd)
  65
  66def isP4Exec(kind):
  67    """Determine if a Perforce 'kind' should have execute permission
  68
  69    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70    or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71    a plus sign, it is also executable"""
  72    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74def setP4ExecBit(file, mode):
  75    # Reopens an already open file and changes the execute bit to match
  76    # the execute bit setting in the passed in mode.
  77
  78    p4Type = "+x"
  79
  80    if not isModeExec(mode):
  81        p4Type = getP4OpenedType(file)
  82        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84        if p4Type[-1] == "+":
  85            p4Type = p4Type[0:-1]
  86
  87    system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89def getP4OpenedType(file):
  90    # Returns the perforce file type for the given file.
  91
  92    result = read_pipe("p4 opened %s" % file)
  93    match = re.match(".*\((.+)\)$", result)
  94    if match:
  95        return match.group(1)
  96    else:
  97        die("Could not determine file type for %s" % file)
  98
  99def diffTreePattern():
 100    # This is a simple generator for the diff tree regex pattern. This could be
 101    # a class variable if this and parseDiffTreeEntry were a part of a class.
 102    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103    while True:
 104        yield pattern
 105
 106def parseDiffTreeEntry(entry):
 107    """Parses a single diff tree entry into its component elements.
 108
 109    See git-diff-tree(1) manpage for details about the format of the diff
 110    output. This method returns a dictionary with the following elements:
 111
 112    src_mode - The mode of the source file
 113    dst_mode - The mode of the destination file
 114    src_sha1 - The sha1 for the source file
 115    dst_sha1 - The sha1 fr the destination file
 116    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117    status_score - The score for the status (applicable for 'C' and 'R'
 118                   statuses). This is None if there is no score.
 119    src - The path for the source file.
 120    dst - The path for the destination file. This is only present for
 121          copy or renames. If it is not present, this is None.
 122
 123    If the pattern is not matched, None is returned."""
 124
 125    match = diffTreePattern().next().match(entry)
 126    if match:
 127        return {
 128            'src_mode': match.group(1),
 129            'dst_mode': match.group(2),
 130            'src_sha1': match.group(3),
 131            'dst_sha1': match.group(4),
 132            'status': match.group(5),
 133            'status_score': match.group(6),
 134            'src': match.group(7),
 135            'dst': match.group(10)
 136        }
 137    return None
 138
 139def isModeExec(mode):
 140    # Returns True if the given git mode represents an executable file,
 141    # otherwise False.
 142    return mode[-3:] == "755"
 143
 144def isModeExecChanged(src_mode, dst_mode):
 145    return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148    cmd = "p4 -G %s" % cmd
 149    if verbose:
 150        sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152    # Use a temporary file to avoid deadlocks without
 153    # subprocess.communicate(), which would put another copy
 154    # of stdout into memory.
 155    stdin_file = None
 156    if stdin is not None:
 157        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158        stdin_file.write(stdin)
 159        stdin_file.flush()
 160        stdin_file.seek(0)
 161
 162    p4 = subprocess.Popen(cmd, shell=True,
 163                          stdin=stdin_file,
 164                          stdout=subprocess.PIPE)
 165
 166    result = []
 167    try:
 168        while True:
 169            entry = marshal.load(p4.stdout)
 170            result.append(entry)
 171    except EOFError:
 172        pass
 173    exitCode = p4.wait()
 174    if exitCode != 0:
 175        entry = {}
 176        entry["p4ExitCode"] = exitCode
 177        result.append(entry)
 178
 179    return result
 180
 181def p4Cmd(cmd):
 182    list = p4CmdList(cmd)
 183    result = {}
 184    for entry in list:
 185        result.update(entry)
 186    return result;
 187
 188def p4Where(depotPath):
 189    if not depotPath.endswith("/"):
 190        depotPath += "/"
 191    output = p4Cmd("where %s..." % depotPath)
 192    if output["code"] == "error":
 193        return ""
 194    clientPath = ""
 195    if "path" in output:
 196        clientPath = output.get("path")
 197    elif "data" in output:
 198        data = output.get("data")
 199        lastSpace = data.rfind(" ")
 200        clientPath = data[lastSpace + 1:]
 201
 202    if clientPath.endswith("..."):
 203        clientPath = clientPath[:-3]
 204    return clientPath
 205
 206def currentGitBranch():
 207    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209def isValidGitDir(path):
 210    if (os.path.exists(path + "/HEAD")
 211        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212        return True;
 213    return False
 214
 215def parseRevision(ref):
 216    return read_pipe("git rev-parse %s" % ref).strip()
 217
 218def extractLogMessageFromGitCommit(commit):
 219    logMessage = ""
 220
 221    ## fixme: title is first line of commit, not 1st paragraph.
 222    foundTitle = False
 223    for log in read_pipe_lines("git cat-file commit %s" % commit):
 224       if not foundTitle:
 225           if len(log) == 1:
 226               foundTitle = True
 227           continue
 228
 229       logMessage += log
 230    return logMessage
 231
 232def extractSettingsGitLog(log):
 233    values = {}
 234    for line in log.split("\n"):
 235        line = line.strip()
 236        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237        if not m:
 238            continue
 239
 240        assignments = m.group(1).split (':')
 241        for a in assignments:
 242            vals = a.split ('=')
 243            key = vals[0].strip()
 244            val = ('='.join (vals[1:])).strip()
 245            if val.endswith ('\"') and val.startswith('"'):
 246                val = val[1:-1]
 247
 248            values[key] = val
 249
 250    paths = values.get("depot-paths")
 251    if not paths:
 252        paths = values.get("depot-path")
 253    if paths:
 254        values['depot-paths'] = paths.split(',')
 255    return values
 256
 257def gitBranchExists(branch):
 258    proc = subprocess.Popen(["git", "rev-parse", branch],
 259                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260    return proc.wait() == 0;
 261
 262def gitConfig(key):
 263    return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265def p4BranchesInGit(branchesAreInRemotes = True):
 266    branches = {}
 267
 268    cmdline = "git rev-parse --symbolic "
 269    if branchesAreInRemotes:
 270        cmdline += " --remotes"
 271    else:
 272        cmdline += " --branches"
 273
 274    for line in read_pipe_lines(cmdline):
 275        line = line.strip()
 276
 277        ## only import to p4/
 278        if not line.startswith('p4/') or line == "p4/HEAD":
 279            continue
 280        branch = line
 281
 282        # strip off p4
 283        branch = re.sub ("^p4/", "", line)
 284
 285        branches[branch] = parseRevision(line)
 286    return branches
 287
 288def findUpstreamBranchPoint(head = "HEAD"):
 289    branches = p4BranchesInGit()
 290    # map from depot-path to branch name
 291    branchByDepotPath = {}
 292    for branch in branches.keys():
 293        tip = branches[branch]
 294        log = extractLogMessageFromGitCommit(tip)
 295        settings = extractSettingsGitLog(log)
 296        if settings.has_key("depot-paths"):
 297            paths = ",".join(settings["depot-paths"])
 298            branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300    settings = None
 301    parent = 0
 302    while parent < 65535:
 303        commit = head + "~%s" % parent
 304        log = extractLogMessageFromGitCommit(commit)
 305        settings = extractSettingsGitLog(log)
 306        if settings.has_key("depot-paths"):
 307            paths = ",".join(settings["depot-paths"])
 308            if branchByDepotPath.has_key(paths):
 309                return [branchByDepotPath[paths], settings]
 310
 311        parent = parent + 1
 312
 313    return ["", settings]
 314
 315def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316    if not silent:
 317        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318               % localRefPrefix)
 319
 320    originPrefix = "origin/p4/"
 321
 322    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323        line = line.strip()
 324        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325            continue
 326
 327        headName = line[len(originPrefix):]
 328        remoteHead = localRefPrefix + headName
 329        originHead = line
 330
 331        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332        if (not original.has_key('depot-paths')
 333            or not original.has_key('change')):
 334            continue
 335
 336        update = False
 337        if not gitBranchExists(remoteHead):
 338            if verbose:
 339                print "creating %s" % remoteHead
 340            update = True
 341        else:
 342            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343            if settings.has_key('change') > 0:
 344                if settings['depot-paths'] == original['depot-paths']:
 345                    originP4Change = int(original['change'])
 346                    p4Change = int(settings['change'])
 347                    if originP4Change > p4Change:
 348                        print ("%s (%s) is newer than %s (%s). "
 349                               "Updating p4 branch from origin."
 350                               % (originHead, originP4Change,
 351                                  remoteHead, p4Change))
 352                        update = True
 353                else:
 354                    print ("Ignoring: %s was imported from %s while "
 355                           "%s was imported from %s"
 356                           % (originHead, ','.join(original['depot-paths']),
 357                              remoteHead, ','.join(settings['depot-paths'])))
 358
 359        if update:
 360            system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362def originP4BranchesExist():
 363        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365def p4ChangesForPaths(depotPaths, changeRange):
 366    assert depotPaths
 367    output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                        for p in depotPaths]))
 369
 370    changes = []
 371    for line in output:
 372        changeNum = line.split(" ")[1]
 373        changes.append(int(changeNum))
 374
 375    changes.sort()
 376    return changes
 377
 378class Command:
 379    def __init__(self):
 380        self.usage = "usage: %prog [options]"
 381        self.needsGit = True
 382
 383class P4Debug(Command):
 384    def __init__(self):
 385        Command.__init__(self)
 386        self.options = [
 387            optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                 default=False),
 389            ]
 390        self.description = "A tool to debug the output of p4 -G."
 391        self.needsGit = False
 392        self.verbose = False
 393
 394    def run(self, args):
 395        j = 0
 396        for output in p4CmdList(" ".join(args)):
 397            print 'Element: %d' % j
 398            j += 1
 399            print output
 400        return True
 401
 402class P4RollBack(Command):
 403    def __init__(self):
 404        Command.__init__(self)
 405        self.options = [
 406            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408        ]
 409        self.description = "A tool to debug the multi-branch import. Don't use :)"
 410        self.verbose = False
 411        self.rollbackLocalBranches = False
 412
 413    def run(self, args):
 414        if len(args) != 1:
 415            return False
 416        maxChange = int(args[0])
 417
 418        if "p4ExitCode" in p4Cmd("changes -m 1"):
 419            die("Problems executing p4");
 420
 421        if self.rollbackLocalBranches:
 422            refPrefix = "refs/heads/"
 423            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424        else:
 425            refPrefix = "refs/remotes/"
 426            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428        for line in lines:
 429            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                line = line.strip()
 431                ref = refPrefix + line
 432                log = extractLogMessageFromGitCommit(ref)
 433                settings = extractSettingsGitLog(log)
 434
 435                depotPaths = settings['depot-paths']
 436                change = settings['change']
 437
 438                changed = False
 439
 440                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                           for p in depotPaths]))) == 0:
 442                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                    continue
 445
 446                while change and int(change) > maxChange:
 447                    changed = True
 448                    if self.verbose:
 449                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                    system("git update-ref %s \"%s^\"" % (ref, ref))
 451                    log = extractLogMessageFromGitCommit(ref)
 452                    settings =  extractSettingsGitLog(log)
 453
 454
 455                    depotPaths = settings['depot-paths']
 456                    change = settings['change']
 457
 458                if changed:
 459                    print "%s rewound to %s" % (ref, change)
 460
 461        return True
 462
 463class P4Submit(Command):
 464    def __init__(self):
 465        Command.__init__(self)
 466        self.options = [
 467                optparse.make_option("--continue", action="store_false", dest="firstTime"),
 468                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 469                optparse.make_option("--origin", dest="origin"),
 470                optparse.make_option("--reset", action="store_true", dest="reset"),
 471                optparse.make_option("-M", dest="detectRename", action="store_true"),
 472        ]
 473        self.description = "Submit changes from git to the perforce depot."
 474        self.usage += " [name of git branch to submit into perforce depot]"
 475        self.firstTime = True
 476        self.reset = False
 477        self.interactive = True
 478        self.firstTime = True
 479        self.origin = ""
 480        self.detectRename = False
 481        self.verbose = False
 482        self.isWindows = (platform.system() == "Windows")
 483
 484    def check(self):
 485        if len(p4CmdList("opened ...")) > 0:
 486            die("You have files opened with perforce! Close them before starting the sync.")
 487
 488    def start(self):
 489        if len(self.config) > 0 and not self.reset:
 490            die("Cannot start sync. Previous sync config found at %s\n"
 491                "If you want to start submitting again from scratch "
 492                "maybe you want to call git-p4 submit --reset" % self.configFile)
 493
 494        commits = []
 495        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 496            commits.append(line.strip())
 497        commits.reverse()
 498
 499        self.config["commits"] = commits
 500
 501    # replaces everything between 'Description:' and the next P4 submit template field with the
 502    # commit message
 503    def prepareLogMessage(self, template, message):
 504        result = ""
 505
 506        inDescriptionSection = False
 507
 508        for line in template.split("\n"):
 509            if line.startswith("#"):
 510                result += line + "\n"
 511                continue
 512
 513            if inDescriptionSection:
 514                if line.startswith("Files:"):
 515                    inDescriptionSection = False
 516                else:
 517                    continue
 518            else:
 519                if line.startswith("Description:"):
 520                    inDescriptionSection = True
 521                    line += "\n"
 522                    for messageLine in message.split("\n"):
 523                        line += "\t" + messageLine + "\n"
 524
 525            result += line + "\n"
 526
 527        return result
 528
 529    def prepareSubmitTemplate(self):
 530        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 531        template = ""
 532        inFilesSection = False
 533        for line in read_pipe_lines("p4 change -o"):
 534            if inFilesSection:
 535                if line.startswith("\t"):
 536                    # path starts and ends with a tab
 537                    path = line[1:]
 538                    lastTab = path.rfind("\t")
 539                    if lastTab != -1:
 540                        path = path[:lastTab]
 541                        if not path.startswith(self.depotPath):
 542                            continue
 543                else:
 544                    inFilesSection = False
 545            else:
 546                if line.startswith("Files:"):
 547                    inFilesSection = True
 548
 549            template += line
 550
 551        return template
 552
 553    def applyCommit(self, id):
 554        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 555        diffOpts = ("", "-M")[self.detectRename]
 556        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 557        filesToAdd = set()
 558        filesToDelete = set()
 559        editedFiles = set()
 560        filesToChangeExecBit = {}
 561        for line in diff:
 562            diff = parseDiffTreeEntry(line)
 563            modifier = diff['status']
 564            path = diff['src']
 565            if modifier == "M":
 566                system("p4 edit \"%s\"" % path)
 567                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 568                    filesToChangeExecBit[path] = diff['dst_mode']
 569                editedFiles.add(path)
 570            elif modifier == "A":
 571                filesToAdd.add(path)
 572                filesToChangeExecBit[path] = diff['dst_mode']
 573                if path in filesToDelete:
 574                    filesToDelete.remove(path)
 575            elif modifier == "D":
 576                filesToDelete.add(path)
 577                if path in filesToAdd:
 578                    filesToAdd.remove(path)
 579            elif modifier == "R":
 580                src, dest = diff['src'], diff['dst']
 581                system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 582                system("p4 edit \"%s\"" % (dest))
 583                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 584                    filesToChangeExecBit[dest] = diff['dst_mode']
 585                os.unlink(dest)
 586                editedFiles.add(dest)
 587                filesToDelete.add(src)
 588            else:
 589                die("unknown modifier %s for %s" % (modifier, path))
 590
 591        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 592        patchcmd = diffcmd + " | git apply "
 593        tryPatchCmd = patchcmd + "--check -"
 594        applyPatchCmd = patchcmd + "--check --apply -"
 595
 596        if os.system(tryPatchCmd) != 0:
 597            print "Unfortunately applying the change failed!"
 598            print "What do you want to do?"
 599            response = "x"
 600            while response != "s" and response != "a" and response != "w":
 601                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 602                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 603            if response == "s":
 604                print "Skipping! Good luck with the next patches..."
 605                for f in editedFiles:
 606                    system("p4 revert \"%s\"" % f);
 607                for f in filesToAdd:
 608                    system("rm %s" %f)
 609                return
 610            elif response == "a":
 611                os.system(applyPatchCmd)
 612                if len(filesToAdd) > 0:
 613                    print "You may also want to call p4 add on the following files:"
 614                    print " ".join(filesToAdd)
 615                if len(filesToDelete):
 616                    print "The following files should be scheduled for deletion with p4 delete:"
 617                    print " ".join(filesToDelete)
 618                die("Please resolve and submit the conflict manually and "
 619                    + "continue afterwards with git-p4 submit --continue")
 620            elif response == "w":
 621                system(diffcmd + " > patch.txt")
 622                print "Patch saved to patch.txt in %s !" % self.clientPath
 623                die("Please resolve and submit the conflict manually and "
 624                    "continue afterwards with git-p4 submit --continue")
 625
 626        system(applyPatchCmd)
 627
 628        for f in filesToAdd:
 629            system("p4 add \"%s\"" % f)
 630        for f in filesToDelete:
 631            system("p4 revert \"%s\"" % f)
 632            system("p4 delete \"%s\"" % f)
 633
 634        # Set/clear executable bits
 635        for f in filesToChangeExecBit.keys():
 636            mode = filesToChangeExecBit[f]
 637            setP4ExecBit(f, mode)
 638
 639        logMessage = extractLogMessageFromGitCommit(id)
 640        if self.isWindows:
 641            logMessage = logMessage.replace("\n", "\r\n")
 642        logMessage = logMessage.strip()
 643
 644        template = self.prepareSubmitTemplate()
 645
 646        if self.interactive:
 647            submitTemplate = self.prepareLogMessage(template, logMessage)
 648            diff = read_pipe("p4 diff -du ...")
 649
 650            for newFile in filesToAdd:
 651                diff += "==== new file ====\n"
 652                diff += "--- /dev/null\n"
 653                diff += "+++ %s\n" % newFile
 654                f = open(newFile, "r")
 655                for line in f.readlines():
 656                    diff += "+" + line
 657                f.close()
 658
 659            separatorLine = "######## everything below this line is just the diff #######"
 660            if platform.system() == "Windows":
 661                separatorLine += "\r"
 662            separatorLine += "\n"
 663
 664            [handle, fileName] = tempfile.mkstemp()
 665            tmpFile = os.fdopen(handle, "w+")
 666            tmpFile.write(submitTemplate + separatorLine + diff)
 667            tmpFile.close()
 668            defaultEditor = "vi"
 669            if platform.system() == "Windows":
 670                defaultEditor = "notepad"
 671            editor = os.environ.get("EDITOR", defaultEditor);
 672            system(editor + " " + fileName)
 673            tmpFile = open(fileName, "rb")
 674            message = tmpFile.read()
 675            tmpFile.close()
 676            os.remove(fileName)
 677            submitTemplate = message[:message.index(separatorLine)]
 678            if self.isWindows:
 679                submitTemplate = submitTemplate.replace("\r\n", "\n")
 680
 681            write_pipe("p4 submit -i", submitTemplate)
 682        else:
 683            fileName = "submit.txt"
 684            file = open(fileName, "w+")
 685            file.write(self.prepareLogMessage(template, logMessage))
 686            file.close()
 687            print ("Perforce submit template written as %s. "
 688                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 689                   % (fileName, fileName))
 690
 691    def run(self, args):
 692        if len(args) == 0:
 693            self.master = currentGitBranch()
 694            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 695                die("Detecting current git branch failed!")
 696        elif len(args) == 1:
 697            self.master = args[0]
 698        else:
 699            return False
 700
 701        [upstream, settings] = findUpstreamBranchPoint()
 702        self.depotPath = settings['depot-paths'][0]
 703        if len(self.origin) == 0:
 704            self.origin = upstream
 705
 706        if self.verbose:
 707            print "Origin branch is " + self.origin
 708
 709        if len(self.depotPath) == 0:
 710            print "Internal error: cannot locate perforce depot path from existing branches"
 711            sys.exit(128)
 712
 713        self.clientPath = p4Where(self.depotPath)
 714
 715        if len(self.clientPath) == 0:
 716            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 717            sys.exit(128)
 718
 719        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 720        self.oldWorkingDirectory = os.getcwd()
 721
 722        os.chdir(self.clientPath)
 723        print "Syncronizing p4 checkout..."
 724        system("p4 sync ...")
 725
 726        if self.reset:
 727            self.firstTime = True
 728
 729        self.check()
 730        self.configFile = self.gitdir + "/p4-git-sync.cfg"
 731        self.config = shelve.open(self.configFile, writeback=True)
 732
 733        if self.firstTime:
 734            self.start()
 735
 736        commits = self.config.get("commits", [])
 737
 738        while len(commits) > 0:
 739            self.firstTime = False
 740            commit = commits[0]
 741            commits = commits[1:]
 742            self.config["commits"] = commits
 743            self.applyCommit(commit)
 744            if not self.interactive:
 745                break
 746
 747        self.config.close()
 748
 749        if len(commits) == 0:
 750            if self.firstTime:
 751                print "No changes found to apply between %s and current HEAD" % self.origin
 752            else:
 753                print "All changes applied!"
 754                os.chdir(self.oldWorkingDirectory)
 755
 756                sync = P4Sync()
 757                sync.run([])
 758
 759                rebase = P4Rebase()
 760                rebase.rebase()
 761            os.remove(self.configFile)
 762
 763        return True
 764
 765class P4Sync(Command):
 766    def __init__(self):
 767        Command.__init__(self)
 768        self.options = [
 769                optparse.make_option("--branch", dest="branch"),
 770                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 771                optparse.make_option("--changesfile", dest="changesFile"),
 772                optparse.make_option("--silent", dest="silent", action="store_true"),
 773                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 774                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 775                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 776                                     help="Import into refs/heads/ , not refs/remotes"),
 777                optparse.make_option("--max-changes", dest="maxChanges"),
 778                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 779                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import")
 780        ]
 781        self.description = """Imports from Perforce into a git repository.\n
 782    example:
 783    //depot/my/project/ -- to import the current head
 784    //depot/my/project/@all -- to import everything
 785    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 786
 787    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 788
 789        self.usage += " //depot/path[@revRange]"
 790        self.silent = False
 791        self.createdBranches = Set()
 792        self.committedChanges = Set()
 793        self.branch = ""
 794        self.detectBranches = False
 795        self.detectLabels = False
 796        self.changesFile = ""
 797        self.syncWithOrigin = True
 798        self.verbose = False
 799        self.importIntoRemotes = True
 800        self.maxChanges = ""
 801        self.isWindows = (platform.system() == "Windows")
 802        self.keepRepoPath = False
 803        self.depotPaths = None
 804        self.p4BranchesInGit = []
 805        self.cloneExclude = []
 806
 807        if gitConfig("git-p4.syncFromOrigin") == "false":
 808            self.syncWithOrigin = False
 809
 810    def extractFilesFromCommit(self, commit):
 811        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 812                             for path in self.cloneExclude]
 813        files = []
 814        fnum = 0
 815        while commit.has_key("depotFile%s" % fnum):
 816            path =  commit["depotFile%s" % fnum]
 817
 818            if [p for p in self.cloneExclude
 819                if path.startswith (p)]:
 820                found = False
 821            else:
 822                found = [p for p in self.depotPaths
 823                         if path.startswith (p)]
 824            if not found:
 825                fnum = fnum + 1
 826                continue
 827
 828            file = {}
 829            file["path"] = path
 830            file["rev"] = commit["rev%s" % fnum]
 831            file["action"] = commit["action%s" % fnum]
 832            file["type"] = commit["type%s" % fnum]
 833            files.append(file)
 834            fnum = fnum + 1
 835        return files
 836
 837    def stripRepoPath(self, path, prefixes):
 838        if self.keepRepoPath:
 839            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 840
 841        for p in prefixes:
 842            if path.startswith(p):
 843                path = path[len(p):]
 844
 845        return path
 846
 847    def splitFilesIntoBranches(self, commit):
 848        branches = {}
 849        fnum = 0
 850        while commit.has_key("depotFile%s" % fnum):
 851            path =  commit["depotFile%s" % fnum]
 852            found = [p for p in self.depotPaths
 853                     if path.startswith (p)]
 854            if not found:
 855                fnum = fnum + 1
 856                continue
 857
 858            file = {}
 859            file["path"] = path
 860            file["rev"] = commit["rev%s" % fnum]
 861            file["action"] = commit["action%s" % fnum]
 862            file["type"] = commit["type%s" % fnum]
 863            fnum = fnum + 1
 864
 865            relPath = self.stripRepoPath(path, self.depotPaths)
 866
 867            for branch in self.knownBranches.keys():
 868
 869                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 870                if relPath.startswith(branch + "/"):
 871                    if branch not in branches:
 872                        branches[branch] = []
 873                    branches[branch].append(file)
 874                    break
 875
 876        return branches
 877
 878    ## Should move this out, doesn't use SELF.
 879    def readP4Files(self, files):
 880        files = [f for f in files
 881                 if f['action'] != 'delete']
 882
 883        if not files:
 884            return
 885
 886        filedata = p4CmdList('-x - print',
 887                             stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 888                                              for f in files]),
 889                             stdin_mode='w+')
 890        if "p4ExitCode" in filedata[0]:
 891            die("Problems executing p4. Error: [%d]."
 892                % (filedata[0]['p4ExitCode']));
 893
 894        j = 0;
 895        contents = {}
 896        while j < len(filedata):
 897            stat = filedata[j]
 898            j += 1
 899            text = ''
 900            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 901                tmp = filedata[j]['data']
 902                if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 903                    tmp = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', tmp)
 904                elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 905                    tmp = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', tmp)
 906                text += tmp
 907                j += 1
 908
 909
 910            if not stat.has_key('depotFile'):
 911                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 912                continue
 913
 914            contents[stat['depotFile']] = text
 915
 916        for f in files:
 917            assert not f.has_key('data')
 918            f['data'] = contents[f['path']]
 919
 920    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 921        epoch = details["time"]
 922        author = details["user"]
 923
 924        if self.verbose:
 925            print "commit into %s" % branch
 926
 927        # start with reading files; if that fails, we should not
 928        # create a commit.
 929        new_files = []
 930        for f in files:
 931            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 932                new_files.append (f)
 933            else:
 934                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 935        files = new_files
 936        self.readP4Files(files)
 937
 938
 939
 940
 941        self.gitStream.write("commit %s\n" % branch)
 942#        gitStream.write("mark :%s\n" % details["change"])
 943        self.committedChanges.add(int(details["change"]))
 944        committer = ""
 945        if author not in self.users:
 946            self.getUserMapFromPerforceServer()
 947        if author in self.users:
 948            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 949        else:
 950            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 951
 952        self.gitStream.write("committer %s\n" % committer)
 953
 954        self.gitStream.write("data <<EOT\n")
 955        self.gitStream.write(details["desc"])
 956        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 957                             % (','.join (branchPrefixes), details["change"]))
 958        if len(details['options']) > 0:
 959            self.gitStream.write(": options = %s" % details['options'])
 960        self.gitStream.write("]\nEOT\n\n")
 961
 962        if len(parent) > 0:
 963            if self.verbose:
 964                print "parent %s" % parent
 965            self.gitStream.write("from %s\n" % parent)
 966
 967        for file in files:
 968            if file["type"] == "apple":
 969                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 970                continue
 971
 972            relPath = self.stripRepoPath(file['path'], branchPrefixes)
 973            if file["action"] == "delete":
 974                self.gitStream.write("D %s\n" % relPath)
 975            else:
 976                data = file['data']
 977
 978                mode = "644"
 979                if isP4Exec(file["type"]):
 980                    mode = "755"
 981                elif file["type"] == "symlink":
 982                    mode = "120000"
 983                    # p4 print on a symlink contains "target\n", so strip it off
 984                    data = data[:-1]
 985
 986                if self.isWindows and file["type"].endswith("text"):
 987                    data = data.replace("\r\n", "\n")
 988
 989                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 990                self.gitStream.write("data %s\n" % len(data))
 991                self.gitStream.write(data)
 992                self.gitStream.write("\n")
 993
 994        self.gitStream.write("\n")
 995
 996        change = int(details["change"])
 997
 998        if self.labels.has_key(change):
 999            label = self.labels[change]
1000            labelDetails = label[0]
1001            labelRevisions = label[1]
1002            if self.verbose:
1003                print "Change %s is labelled %s" % (change, labelDetails)
1004
1005            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1006                                                    for p in branchPrefixes]))
1007
1008            if len(files) == len(labelRevisions):
1009
1010                cleanedFiles = {}
1011                for info in files:
1012                    if info["action"] == "delete":
1013                        continue
1014                    cleanedFiles[info["depotFile"]] = info["rev"]
1015
1016                if cleanedFiles == labelRevisions:
1017                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1018                    self.gitStream.write("from %s\n" % branch)
1019
1020                    owner = labelDetails["Owner"]
1021                    tagger = ""
1022                    if author in self.users:
1023                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1024                    else:
1025                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1026                    self.gitStream.write("tagger %s\n" % tagger)
1027                    self.gitStream.write("data <<EOT\n")
1028                    self.gitStream.write(labelDetails["Description"])
1029                    self.gitStream.write("EOT\n\n")
1030
1031                else:
1032                    if not self.silent:
1033                        print ("Tag %s does not match with change %s: files do not match."
1034                               % (labelDetails["label"], change))
1035
1036            else:
1037                if not self.silent:
1038                    print ("Tag %s does not match with change %s: file count is different."
1039                           % (labelDetails["label"], change))
1040
1041    def getUserCacheFilename(self):
1042        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1043        return home + "/.gitp4-usercache.txt"
1044
1045    def getUserMapFromPerforceServer(self):
1046        if self.userMapFromPerforceServer:
1047            return
1048        self.users = {}
1049
1050        for output in p4CmdList("users"):
1051            if not output.has_key("User"):
1052                continue
1053            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1054
1055
1056        s = ''
1057        for (key, val) in self.users.items():
1058            s += "%s\t%s\n" % (key, val)
1059
1060        open(self.getUserCacheFilename(), "wb").write(s)
1061        self.userMapFromPerforceServer = True
1062
1063    def loadUserMapFromCache(self):
1064        self.users = {}
1065        self.userMapFromPerforceServer = False
1066        try:
1067            cache = open(self.getUserCacheFilename(), "rb")
1068            lines = cache.readlines()
1069            cache.close()
1070            for line in lines:
1071                entry = line.strip().split("\t")
1072                self.users[entry[0]] = entry[1]
1073        except IOError:
1074            self.getUserMapFromPerforceServer()
1075
1076    def getLabels(self):
1077        self.labels = {}
1078
1079        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1080        if len(l) > 0 and not self.silent:
1081            print "Finding files belonging to labels in %s" % `self.depotPaths`
1082
1083        for output in l:
1084            label = output["label"]
1085            revisions = {}
1086            newestChange = 0
1087            if self.verbose:
1088                print "Querying files for label %s" % label
1089            for file in p4CmdList("files "
1090                                  +  ' '.join (["%s...@%s" % (p, label)
1091                                                for p in self.depotPaths])):
1092                revisions[file["depotFile"]] = file["rev"]
1093                change = int(file["change"])
1094                if change > newestChange:
1095                    newestChange = change
1096
1097            self.labels[newestChange] = [output, revisions]
1098
1099        if self.verbose:
1100            print "Label changes: %s" % self.labels.keys()
1101
1102    def guessProjectName(self):
1103        for p in self.depotPaths:
1104            if p.endswith("/"):
1105                p = p[:-1]
1106            p = p[p.strip().rfind("/") + 1:]
1107            if not p.endswith("/"):
1108               p += "/"
1109            return p
1110
1111    def getBranchMapping(self):
1112        lostAndFoundBranches = set()
1113
1114        for info in p4CmdList("branches"):
1115            details = p4Cmd("branch -o %s" % info["branch"])
1116            viewIdx = 0
1117            while details.has_key("View%s" % viewIdx):
1118                paths = details["View%s" % viewIdx].split(" ")
1119                viewIdx = viewIdx + 1
1120                # require standard //depot/foo/... //depot/bar/... mapping
1121                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1122                    continue
1123                source = paths[0]
1124                destination = paths[1]
1125                ## HACK
1126                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1127                    source = source[len(self.depotPaths[0]):-4]
1128                    destination = destination[len(self.depotPaths[0]):-4]
1129
1130                    if destination in self.knownBranches:
1131                        if not self.silent:
1132                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1133                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1134                        continue
1135
1136                    self.knownBranches[destination] = source
1137
1138                    lostAndFoundBranches.discard(destination)
1139
1140                    if source not in self.knownBranches:
1141                        lostAndFoundBranches.add(source)
1142
1143
1144        for branch in lostAndFoundBranches:
1145            self.knownBranches[branch] = branch
1146
1147    def getBranchMappingFromGitBranches(self):
1148        branches = p4BranchesInGit(self.importIntoRemotes)
1149        for branch in branches.keys():
1150            if branch == "master":
1151                branch = "main"
1152            else:
1153                branch = branch[len(self.projectName):]
1154            self.knownBranches[branch] = branch
1155
1156    def listExistingP4GitBranches(self):
1157        # branches holds mapping from name to commit
1158        branches = p4BranchesInGit(self.importIntoRemotes)
1159        self.p4BranchesInGit = branches.keys()
1160        for branch in branches.keys():
1161            self.initialParents[self.refPrefix + branch] = branches[branch]
1162
1163    def updateOptionDict(self, d):
1164        option_keys = {}
1165        if self.keepRepoPath:
1166            option_keys['keepRepoPath'] = 1
1167
1168        d["options"] = ' '.join(sorted(option_keys.keys()))
1169
1170    def readOptions(self, d):
1171        self.keepRepoPath = (d.has_key('options')
1172                             and ('keepRepoPath' in d['options']))
1173
1174    def gitRefForBranch(self, branch):
1175        if branch == "main":
1176            return self.refPrefix + "master"
1177
1178        if len(branch) <= 0:
1179            return branch
1180
1181        return self.refPrefix + self.projectName + branch
1182
1183    def gitCommitByP4Change(self, ref, change):
1184        if self.verbose:
1185            print "looking in ref " + ref + " for change %s using bisect..." % change
1186
1187        earliestCommit = ""
1188        latestCommit = parseRevision(ref)
1189
1190        while True:
1191            if self.verbose:
1192                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1193            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1194            if len(next) == 0:
1195                if self.verbose:
1196                    print "argh"
1197                return ""
1198            log = extractLogMessageFromGitCommit(next)
1199            settings = extractSettingsGitLog(log)
1200            currentChange = int(settings['change'])
1201            if self.verbose:
1202                print "current change %s" % currentChange
1203
1204            if currentChange == change:
1205                if self.verbose:
1206                    print "found %s" % next
1207                return next
1208
1209            if currentChange < change:
1210                earliestCommit = "^%s" % next
1211            else:
1212                latestCommit = "%s" % next
1213
1214        return ""
1215
1216    def importNewBranch(self, branch, maxChange):
1217        # make fast-import flush all changes to disk and update the refs using the checkpoint
1218        # command so that we can try to find the branch parent in the git history
1219        self.gitStream.write("checkpoint\n\n");
1220        self.gitStream.flush();
1221        branchPrefix = self.depotPaths[0] + branch + "/"
1222        range = "@1,%s" % maxChange
1223        #print "prefix" + branchPrefix
1224        changes = p4ChangesForPaths([branchPrefix], range)
1225        if len(changes) <= 0:
1226            return False
1227        firstChange = changes[0]
1228        #print "first change in branch: %s" % firstChange
1229        sourceBranch = self.knownBranches[branch]
1230        sourceDepotPath = self.depotPaths[0] + sourceBranch
1231        sourceRef = self.gitRefForBranch(sourceBranch)
1232        #print "source " + sourceBranch
1233
1234        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1235        #print "branch parent: %s" % branchParentChange
1236        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1237        if len(gitParent) > 0:
1238            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1239            #print "parent git commit: %s" % gitParent
1240
1241        self.importChanges(changes)
1242        return True
1243
1244    def importChanges(self, changes):
1245        cnt = 1
1246        for change in changes:
1247            description = p4Cmd("describe %s" % change)
1248            self.updateOptionDict(description)
1249
1250            if not self.silent:
1251                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1252                sys.stdout.flush()
1253            cnt = cnt + 1
1254
1255            try:
1256                if self.detectBranches:
1257                    branches = self.splitFilesIntoBranches(description)
1258                    for branch in branches.keys():
1259                        ## HACK  --hwn
1260                        branchPrefix = self.depotPaths[0] + branch + "/"
1261
1262                        parent = ""
1263
1264                        filesForCommit = branches[branch]
1265
1266                        if self.verbose:
1267                            print "branch is %s" % branch
1268
1269                        self.updatedBranches.add(branch)
1270
1271                        if branch not in self.createdBranches:
1272                            self.createdBranches.add(branch)
1273                            parent = self.knownBranches[branch]
1274                            if parent == branch:
1275                                parent = ""
1276                            else:
1277                                fullBranch = self.projectName + branch
1278                                if fullBranch not in self.p4BranchesInGit:
1279                                    if not self.silent:
1280                                        print("\n    Importing new branch %s" % fullBranch);
1281                                    if self.importNewBranch(branch, change - 1):
1282                                        parent = ""
1283                                        self.p4BranchesInGit.append(fullBranch)
1284                                    if not self.silent:
1285                                        print("\n    Resuming with change %s" % change);
1286
1287                                if self.verbose:
1288                                    print "parent determined through known branches: %s" % parent
1289
1290                        branch = self.gitRefForBranch(branch)
1291                        parent = self.gitRefForBranch(parent)
1292
1293                        if self.verbose:
1294                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1295
1296                        if len(parent) == 0 and branch in self.initialParents:
1297                            parent = self.initialParents[branch]
1298                            del self.initialParents[branch]
1299
1300                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1301                else:
1302                    files = self.extractFilesFromCommit(description)
1303                    self.commit(description, files, self.branch, self.depotPaths,
1304                                self.initialParent)
1305                    self.initialParent = ""
1306            except IOError:
1307                print self.gitError.read()
1308                sys.exit(1)
1309
1310    def importHeadRevision(self, revision):
1311        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1312
1313        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1314        details["desc"] = ("Initial import of %s from the state at revision %s"
1315                           % (' '.join(self.depotPaths), revision))
1316        details["change"] = revision
1317        newestRevision = 0
1318
1319        fileCnt = 0
1320        for info in p4CmdList("files "
1321                              +  ' '.join(["%s...%s"
1322                                           % (p, revision)
1323                                           for p in self.depotPaths])):
1324
1325            if info['code'] == 'error':
1326                sys.stderr.write("p4 returned an error: %s\n"
1327                                 % info['data'])
1328                sys.exit(1)
1329
1330
1331            change = int(info["change"])
1332            if change > newestRevision:
1333                newestRevision = change
1334
1335            if info["action"] == "delete":
1336                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1337                #fileCnt = fileCnt + 1
1338                continue
1339
1340            for prop in ["depotFile", "rev", "action", "type" ]:
1341                details["%s%s" % (prop, fileCnt)] = info[prop]
1342
1343            fileCnt = fileCnt + 1
1344
1345        details["change"] = newestRevision
1346        self.updateOptionDict(details)
1347        try:
1348            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1349        except IOError:
1350            print "IO error with git fast-import. Is your git version recent enough?"
1351            print self.gitError.read()
1352
1353
1354    def run(self, args):
1355        self.depotPaths = []
1356        self.changeRange = ""
1357        self.initialParent = ""
1358        self.previousDepotPaths = []
1359
1360        # map from branch depot path to parent branch
1361        self.knownBranches = {}
1362        self.initialParents = {}
1363        self.hasOrigin = originP4BranchesExist()
1364        if not self.syncWithOrigin:
1365            self.hasOrigin = False
1366
1367        if self.importIntoRemotes:
1368            self.refPrefix = "refs/remotes/p4/"
1369        else:
1370            self.refPrefix = "refs/heads/p4/"
1371
1372        if self.syncWithOrigin and self.hasOrigin:
1373            if not self.silent:
1374                print "Syncing with origin first by calling git fetch origin"
1375            system("git fetch origin")
1376
1377        if len(self.branch) == 0:
1378            self.branch = self.refPrefix + "master"
1379            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1380                system("git update-ref %s refs/heads/p4" % self.branch)
1381                system("git branch -D p4");
1382            # create it /after/ importing, when master exists
1383            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1384                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1385
1386        # TODO: should always look at previous commits,
1387        # merge with previous imports, if possible.
1388        if args == []:
1389            if self.hasOrigin:
1390                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1391            self.listExistingP4GitBranches()
1392
1393            if len(self.p4BranchesInGit) > 1:
1394                if not self.silent:
1395                    print "Importing from/into multiple branches"
1396                self.detectBranches = True
1397
1398            if self.verbose:
1399                print "branches: %s" % self.p4BranchesInGit
1400
1401            p4Change = 0
1402            for branch in self.p4BranchesInGit:
1403                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1404
1405                settings = extractSettingsGitLog(logMsg)
1406
1407                self.readOptions(settings)
1408                if (settings.has_key('depot-paths')
1409                    and settings.has_key ('change')):
1410                    change = int(settings['change']) + 1
1411                    p4Change = max(p4Change, change)
1412
1413                    depotPaths = sorted(settings['depot-paths'])
1414                    if self.previousDepotPaths == []:
1415                        self.previousDepotPaths = depotPaths
1416                    else:
1417                        paths = []
1418                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1419                            for i in range(0, min(len(cur), len(prev))):
1420                                if cur[i] <> prev[i]:
1421                                    i = i - 1
1422                                    break
1423
1424                            paths.append (cur[:i + 1])
1425
1426                        self.previousDepotPaths = paths
1427
1428            if p4Change > 0:
1429                self.depotPaths = sorted(self.previousDepotPaths)
1430                self.changeRange = "@%s,#head" % p4Change
1431                if not self.detectBranches:
1432                    self.initialParent = parseRevision(self.branch)
1433                if not self.silent and not self.detectBranches:
1434                    print "Performing incremental import into %s git branch" % self.branch
1435
1436        if not self.branch.startswith("refs/"):
1437            self.branch = "refs/heads/" + self.branch
1438
1439        if len(args) == 0 and self.depotPaths:
1440            if not self.silent:
1441                print "Depot paths: %s" % ' '.join(self.depotPaths)
1442        else:
1443            if self.depotPaths and self.depotPaths != args:
1444                print ("previous import used depot path %s and now %s was specified. "
1445                       "This doesn't work!" % (' '.join (self.depotPaths),
1446                                               ' '.join (args)))
1447                sys.exit(1)
1448
1449            self.depotPaths = sorted(args)
1450
1451        revision = ""
1452        self.users = {}
1453
1454        newPaths = []
1455        for p in self.depotPaths:
1456            if p.find("@") != -1:
1457                atIdx = p.index("@")
1458                self.changeRange = p[atIdx:]
1459                if self.changeRange == "@all":
1460                    self.changeRange = ""
1461                elif ',' not in self.changeRange:
1462                    revision = self.changeRange
1463                    self.changeRange = ""
1464                p = p[:atIdx]
1465            elif p.find("#") != -1:
1466                hashIdx = p.index("#")
1467                revision = p[hashIdx:]
1468                p = p[:hashIdx]
1469            elif self.previousDepotPaths == []:
1470                revision = "#head"
1471
1472            p = re.sub ("\.\.\.$", "", p)
1473            if not p.endswith("/"):
1474                p += "/"
1475
1476            newPaths.append(p)
1477
1478        self.depotPaths = newPaths
1479
1480
1481        self.loadUserMapFromCache()
1482        self.labels = {}
1483        if self.detectLabels:
1484            self.getLabels();
1485
1486        if self.detectBranches:
1487            ## FIXME - what's a P4 projectName ?
1488            self.projectName = self.guessProjectName()
1489
1490            if self.hasOrigin:
1491                self.getBranchMappingFromGitBranches()
1492            else:
1493                self.getBranchMapping()
1494            if self.verbose:
1495                print "p4-git branches: %s" % self.p4BranchesInGit
1496                print "initial parents: %s" % self.initialParents
1497            for b in self.p4BranchesInGit:
1498                if b != "master":
1499
1500                    ## FIXME
1501                    b = b[len(self.projectName):]
1502                self.createdBranches.add(b)
1503
1504        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1505
1506        importProcess = subprocess.Popen(["git", "fast-import"],
1507                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1508                                         stderr=subprocess.PIPE);
1509        self.gitOutput = importProcess.stdout
1510        self.gitStream = importProcess.stdin
1511        self.gitError = importProcess.stderr
1512
1513        if revision:
1514            self.importHeadRevision(revision)
1515        else:
1516            changes = []
1517
1518            if len(self.changesFile) > 0:
1519                output = open(self.changesFile).readlines()
1520                changeSet = Set()
1521                for line in output:
1522                    changeSet.add(int(line))
1523
1524                for change in changeSet:
1525                    changes.append(change)
1526
1527                changes.sort()
1528            else:
1529                if self.verbose:
1530                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1531                                                              self.changeRange)
1532                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1533
1534                if len(self.maxChanges) > 0:
1535                    changes = changes[:min(int(self.maxChanges), len(changes))]
1536
1537            if len(changes) == 0:
1538                if not self.silent:
1539                    print "No changes to import!"
1540                return True
1541
1542            if not self.silent and not self.detectBranches:
1543                print "Import destination: %s" % self.branch
1544
1545            self.updatedBranches = set()
1546
1547            self.importChanges(changes)
1548
1549            if not self.silent:
1550                print ""
1551                if len(self.updatedBranches) > 0:
1552                    sys.stdout.write("Updated branches: ")
1553                    for b in self.updatedBranches:
1554                        sys.stdout.write("%s " % b)
1555                    sys.stdout.write("\n")
1556
1557        self.gitStream.close()
1558        if importProcess.wait() != 0:
1559            die("fast-import failed: %s" % self.gitError.read())
1560        self.gitOutput.close()
1561        self.gitError.close()
1562
1563        return True
1564
1565class P4Rebase(Command):
1566    def __init__(self):
1567        Command.__init__(self)
1568        self.options = [ ]
1569        self.description = ("Fetches the latest revision from perforce and "
1570                            + "rebases the current work (branch) against it")
1571        self.verbose = False
1572
1573    def run(self, args):
1574        sync = P4Sync()
1575        sync.run([])
1576
1577        return self.rebase()
1578
1579    def rebase(self):
1580        if os.system("git update-index --refresh") != 0:
1581            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1582        if len(read_pipe("git diff-index HEAD --")) > 0:
1583            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1584
1585        [upstream, settings] = findUpstreamBranchPoint()
1586        if len(upstream) == 0:
1587            die("Cannot find upstream branchpoint for rebase")
1588
1589        # the branchpoint may be p4/foo~3, so strip off the parent
1590        upstream = re.sub("~[0-9]+$", "", upstream)
1591
1592        print "Rebasing the current branch onto %s" % upstream
1593        oldHead = read_pipe("git rev-parse HEAD").strip()
1594        system("git rebase %s" % upstream)
1595        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1596        return True
1597
1598class P4Clone(P4Sync):
1599    def __init__(self):
1600        P4Sync.__init__(self)
1601        self.description = "Creates a new git repository and imports from Perforce into it"
1602        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1603        self.options += [
1604            optparse.make_option("--destination", dest="cloneDestination",
1605                                 action='store', default=None,
1606                                 help="where to leave result of the clone"),
1607            optparse.make_option("-/", dest="cloneExclude",
1608                                 action="append", type="string",
1609                                 help="exclude depot path")
1610        ]
1611        self.cloneDestination = None
1612        self.needsGit = False
1613
1614    # This is required for the "append" cloneExclude action
1615    def ensure_value(self, attr, value):
1616        if not hasattr(self, attr) or getattr(self, attr) is None:
1617            setattr(self, attr, value)
1618        return getattr(self, attr)
1619
1620    def defaultDestination(self, args):
1621        ## TODO: use common prefix of args?
1622        depotPath = args[0]
1623        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1624        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1625        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1626        depotDir = re.sub(r"/$", "", depotDir)
1627        return os.path.split(depotDir)[1]
1628
1629    def run(self, args):
1630        if len(args) < 1:
1631            return False
1632
1633        if self.keepRepoPath and not self.cloneDestination:
1634            sys.stderr.write("Must specify destination for --keep-path\n")
1635            sys.exit(1)
1636
1637        depotPaths = args
1638
1639        if not self.cloneDestination and len(depotPaths) > 1:
1640            self.cloneDestination = depotPaths[-1]
1641            depotPaths = depotPaths[:-1]
1642
1643        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1644        for p in depotPaths:
1645            if not p.startswith("//"):
1646                return False
1647
1648        if not self.cloneDestination:
1649            self.cloneDestination = self.defaultDestination(args)
1650
1651        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1652        if not os.path.exists(self.cloneDestination):
1653            os.makedirs(self.cloneDestination)
1654        os.chdir(self.cloneDestination)
1655        system("git init")
1656        self.gitdir = os.getcwd() + "/.git"
1657        if not P4Sync.run(self, depotPaths):
1658            return False
1659        if self.branch != "master":
1660            if gitBranchExists("refs/remotes/p4/master"):
1661                system("git branch master refs/remotes/p4/master")
1662                system("git checkout -f")
1663            else:
1664                print "Could not detect main branch. No checkout/master branch created."
1665
1666        return True
1667
1668class P4Branches(Command):
1669    def __init__(self):
1670        Command.__init__(self)
1671        self.options = [ ]
1672        self.description = ("Shows the git branches that hold imports and their "
1673                            + "corresponding perforce depot paths")
1674        self.verbose = False
1675
1676    def run(self, args):
1677        if originP4BranchesExist():
1678            createOrUpdateBranchesFromOrigin()
1679
1680        cmdline = "git rev-parse --symbolic "
1681        cmdline += " --remotes"
1682
1683        for line in read_pipe_lines(cmdline):
1684            line = line.strip()
1685
1686            if not line.startswith('p4/') or line == "p4/HEAD":
1687                continue
1688            branch = line
1689
1690            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1691            settings = extractSettingsGitLog(log)
1692
1693            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1694        return True
1695
1696class HelpFormatter(optparse.IndentedHelpFormatter):
1697    def __init__(self):
1698        optparse.IndentedHelpFormatter.__init__(self)
1699
1700    def format_description(self, description):
1701        if description:
1702            return description + "\n"
1703        else:
1704            return ""
1705
1706def printUsage(commands):
1707    print "usage: %s <command> [options]" % sys.argv[0]
1708    print ""
1709    print "valid commands: %s" % ", ".join(commands)
1710    print ""
1711    print "Try %s <command> --help for command specific help." % sys.argv[0]
1712    print ""
1713
1714commands = {
1715    "debug" : P4Debug,
1716    "submit" : P4Submit,
1717    "commit" : P4Submit,
1718    "sync" : P4Sync,
1719    "rebase" : P4Rebase,
1720    "clone" : P4Clone,
1721    "rollback" : P4RollBack,
1722    "branches" : P4Branches
1723}
1724
1725
1726def main():
1727    if len(sys.argv[1:]) == 0:
1728        printUsage(commands.keys())
1729        sys.exit(2)
1730
1731    cmd = ""
1732    cmdName = sys.argv[1]
1733    try:
1734        klass = commands[cmdName]
1735        cmd = klass()
1736    except KeyError:
1737        print "unknown command %s" % cmdName
1738        print ""
1739        printUsage(commands.keys())
1740        sys.exit(2)
1741
1742    options = cmd.options
1743    cmd.gitdir = os.environ.get("GIT_DIR", None)
1744
1745    args = sys.argv[2:]
1746
1747    if len(options) > 0:
1748        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1749
1750        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1751                                       options,
1752                                       description = cmd.description,
1753                                       formatter = HelpFormatter())
1754
1755        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1756    global verbose
1757    verbose = cmd.verbose
1758    if cmd.needsGit:
1759        if cmd.gitdir == None:
1760            cmd.gitdir = os.path.abspath(".git")
1761            if not isValidGitDir(cmd.gitdir):
1762                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1763                if os.path.exists(cmd.gitdir):
1764                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1765                    if len(cdup) > 0:
1766                        os.chdir(cdup);
1767
1768        if not isValidGitDir(cmd.gitdir):
1769            if isValidGitDir(cmd.gitdir + "/.git"):
1770                cmd.gitdir += "/.git"
1771            else:
1772                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1773
1774        os.environ["GIT_DIR"] = cmd.gitdir
1775
1776    if not cmd.run(args):
1777        parser.print_help()
1778
1779
1780if __name__ == '__main__':
1781    main()