Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'dd/cvsserver' (22e885e)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19def die(msg):
  20    if verbose:
  21        raise Exception(msg)
  22    else:
  23        sys.stderr.write(msg + "\n")
  24        sys.exit(1)
  25
  26def write_pipe(c, str):
  27    if verbose:
  28        sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30    pipe = os.popen(c, 'w')
  31    val = pipe.write(str)
  32    if pipe.close():
  33        die('Command failed: %s' % c)
  34
  35    return val
  36
  37def read_pipe(c, ignore_error=False):
  38    if verbose:
  39        sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41    pipe = os.popen(c, 'rb')
  42    val = pipe.read()
  43    if pipe.close() and not ignore_error:
  44        die('Command failed: %s' % c)
  45
  46    return val
  47
  48
  49def read_pipe_lines(c):
  50    if verbose:
  51        sys.stderr.write('Reading pipe: %s\n' % c)
  52    ## todo: check return status
  53    pipe = os.popen(c, 'rb')
  54    val = pipe.readlines()
  55    if pipe.close():
  56        die('Command failed: %s' % c)
  57
  58    return val
  59
  60def system(cmd):
  61    if verbose:
  62        sys.stderr.write("executing %s\n" % cmd)
  63    if os.system(cmd) != 0:
  64        die("command failed: %s" % cmd)
  65
  66def isP4Exec(kind):
  67    """Determine if a Perforce 'kind' should have execute permission
  68
  69    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70    or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71    a plus sign, it is also executable"""
  72    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74def setP4ExecBit(file, mode):
  75    # Reopens an already open file and changes the execute bit to match
  76    # the execute bit setting in the passed in mode.
  77
  78    p4Type = "+x"
  79
  80    if not isModeExec(mode):
  81        p4Type = getP4OpenedType(file)
  82        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84        if p4Type[-1] == "+":
  85            p4Type = p4Type[0:-1]
  86
  87    system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89def getP4OpenedType(file):
  90    # Returns the perforce file type for the given file.
  91
  92    result = read_pipe("p4 opened %s" % file)
  93    match = re.match(".*\((.+)\)\r?$", result)
  94    if match:
  95        return match.group(1)
  96    else:
  97        die("Could not determine file type for %s (result: '%s')" % (file, result))
  98
  99def diffTreePattern():
 100    # This is a simple generator for the diff tree regex pattern. This could be
 101    # a class variable if this and parseDiffTreeEntry were a part of a class.
 102    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103    while True:
 104        yield pattern
 105
 106def parseDiffTreeEntry(entry):
 107    """Parses a single diff tree entry into its component elements.
 108
 109    See git-diff-tree(1) manpage for details about the format of the diff
 110    output. This method returns a dictionary with the following elements:
 111
 112    src_mode - The mode of the source file
 113    dst_mode - The mode of the destination file
 114    src_sha1 - The sha1 for the source file
 115    dst_sha1 - The sha1 fr the destination file
 116    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117    status_score - The score for the status (applicable for 'C' and 'R'
 118                   statuses). This is None if there is no score.
 119    src - The path for the source file.
 120    dst - The path for the destination file. This is only present for
 121          copy or renames. If it is not present, this is None.
 122
 123    If the pattern is not matched, None is returned."""
 124
 125    match = diffTreePattern().next().match(entry)
 126    if match:
 127        return {
 128            'src_mode': match.group(1),
 129            'dst_mode': match.group(2),
 130            'src_sha1': match.group(3),
 131            'dst_sha1': match.group(4),
 132            'status': match.group(5),
 133            'status_score': match.group(6),
 134            'src': match.group(7),
 135            'dst': match.group(10)
 136        }
 137    return None
 138
 139def isModeExec(mode):
 140    # Returns True if the given git mode represents an executable file,
 141    # otherwise False.
 142    return mode[-3:] == "755"
 143
 144def isModeExecChanged(src_mode, dst_mode):
 145    return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148    cmd = "p4 -G %s" % cmd
 149    if verbose:
 150        sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152    # Use a temporary file to avoid deadlocks without
 153    # subprocess.communicate(), which would put another copy
 154    # of stdout into memory.
 155    stdin_file = None
 156    if stdin is not None:
 157        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158        stdin_file.write(stdin)
 159        stdin_file.flush()
 160        stdin_file.seek(0)
 161
 162    p4 = subprocess.Popen(cmd, shell=True,
 163                          stdin=stdin_file,
 164                          stdout=subprocess.PIPE)
 165
 166    result = []
 167    try:
 168        while True:
 169            entry = marshal.load(p4.stdout)
 170            result.append(entry)
 171    except EOFError:
 172        pass
 173    exitCode = p4.wait()
 174    if exitCode != 0:
 175        entry = {}
 176        entry["p4ExitCode"] = exitCode
 177        result.append(entry)
 178
 179    return result
 180
 181def p4Cmd(cmd):
 182    list = p4CmdList(cmd)
 183    result = {}
 184    for entry in list:
 185        result.update(entry)
 186    return result;
 187
 188def p4Where(depotPath):
 189    if not depotPath.endswith("/"):
 190        depotPath += "/"
 191    output = p4Cmd("where %s..." % depotPath)
 192    if output["code"] == "error":
 193        return ""
 194    clientPath = ""
 195    if "path" in output:
 196        clientPath = output.get("path")
 197    elif "data" in output:
 198        data = output.get("data")
 199        lastSpace = data.rfind(" ")
 200        clientPath = data[lastSpace + 1:]
 201
 202    if clientPath.endswith("..."):
 203        clientPath = clientPath[:-3]
 204    return clientPath
 205
 206def currentGitBranch():
 207    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209def isValidGitDir(path):
 210    if (os.path.exists(path + "/HEAD")
 211        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212        return True;
 213    return False
 214
 215def parseRevision(ref):
 216    return read_pipe("git rev-parse %s" % ref).strip()
 217
 218def extractLogMessageFromGitCommit(commit):
 219    logMessage = ""
 220
 221    ## fixme: title is first line of commit, not 1st paragraph.
 222    foundTitle = False
 223    for log in read_pipe_lines("git cat-file commit %s" % commit):
 224       if not foundTitle:
 225           if len(log) == 1:
 226               foundTitle = True
 227           continue
 228
 229       logMessage += log
 230    return logMessage
 231
 232def extractSettingsGitLog(log):
 233    values = {}
 234    for line in log.split("\n"):
 235        line = line.strip()
 236        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237        if not m:
 238            continue
 239
 240        assignments = m.group(1).split (':')
 241        for a in assignments:
 242            vals = a.split ('=')
 243            key = vals[0].strip()
 244            val = ('='.join (vals[1:])).strip()
 245            if val.endswith ('\"') and val.startswith('"'):
 246                val = val[1:-1]
 247
 248            values[key] = val
 249
 250    paths = values.get("depot-paths")
 251    if not paths:
 252        paths = values.get("depot-path")
 253    if paths:
 254        values['depot-paths'] = paths.split(',')
 255    return values
 256
 257def gitBranchExists(branch):
 258    proc = subprocess.Popen(["git", "rev-parse", branch],
 259                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260    return proc.wait() == 0;
 261
 262def gitConfig(key):
 263    return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265def p4BranchesInGit(branchesAreInRemotes = True):
 266    branches = {}
 267
 268    cmdline = "git rev-parse --symbolic "
 269    if branchesAreInRemotes:
 270        cmdline += " --remotes"
 271    else:
 272        cmdline += " --branches"
 273
 274    for line in read_pipe_lines(cmdline):
 275        line = line.strip()
 276
 277        ## only import to p4/
 278        if not line.startswith('p4/') or line == "p4/HEAD":
 279            continue
 280        branch = line
 281
 282        # strip off p4
 283        branch = re.sub ("^p4/", "", line)
 284
 285        branches[branch] = parseRevision(line)
 286    return branches
 287
 288def findUpstreamBranchPoint(head = "HEAD"):
 289    branches = p4BranchesInGit()
 290    # map from depot-path to branch name
 291    branchByDepotPath = {}
 292    for branch in branches.keys():
 293        tip = branches[branch]
 294        log = extractLogMessageFromGitCommit(tip)
 295        settings = extractSettingsGitLog(log)
 296        if settings.has_key("depot-paths"):
 297            paths = ",".join(settings["depot-paths"])
 298            branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300    settings = None
 301    parent = 0
 302    while parent < 65535:
 303        commit = head + "~%s" % parent
 304        log = extractLogMessageFromGitCommit(commit)
 305        settings = extractSettingsGitLog(log)
 306        if settings.has_key("depot-paths"):
 307            paths = ",".join(settings["depot-paths"])
 308            if branchByDepotPath.has_key(paths):
 309                return [branchByDepotPath[paths], settings]
 310
 311        parent = parent + 1
 312
 313    return ["", settings]
 314
 315def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316    if not silent:
 317        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318               % localRefPrefix)
 319
 320    originPrefix = "origin/p4/"
 321
 322    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323        line = line.strip()
 324        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325            continue
 326
 327        headName = line[len(originPrefix):]
 328        remoteHead = localRefPrefix + headName
 329        originHead = line
 330
 331        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332        if (not original.has_key('depot-paths')
 333            or not original.has_key('change')):
 334            continue
 335
 336        update = False
 337        if not gitBranchExists(remoteHead):
 338            if verbose:
 339                print "creating %s" % remoteHead
 340            update = True
 341        else:
 342            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343            if settings.has_key('change') > 0:
 344                if settings['depot-paths'] == original['depot-paths']:
 345                    originP4Change = int(original['change'])
 346                    p4Change = int(settings['change'])
 347                    if originP4Change > p4Change:
 348                        print ("%s (%s) is newer than %s (%s). "
 349                               "Updating p4 branch from origin."
 350                               % (originHead, originP4Change,
 351                                  remoteHead, p4Change))
 352                        update = True
 353                else:
 354                    print ("Ignoring: %s was imported from %s while "
 355                           "%s was imported from %s"
 356                           % (originHead, ','.join(original['depot-paths']),
 357                              remoteHead, ','.join(settings['depot-paths'])))
 358
 359        if update:
 360            system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362def originP4BranchesExist():
 363        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365def p4ChangesForPaths(depotPaths, changeRange):
 366    assert depotPaths
 367    output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                        for p in depotPaths]))
 369
 370    changes = []
 371    for line in output:
 372        changeNum = line.split(" ")[1]
 373        changes.append(int(changeNum))
 374
 375    changes.sort()
 376    return changes
 377
 378class Command:
 379    def __init__(self):
 380        self.usage = "usage: %prog [options]"
 381        self.needsGit = True
 382
 383class P4Debug(Command):
 384    def __init__(self):
 385        Command.__init__(self)
 386        self.options = [
 387            optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                 default=False),
 389            ]
 390        self.description = "A tool to debug the output of p4 -G."
 391        self.needsGit = False
 392        self.verbose = False
 393
 394    def run(self, args):
 395        j = 0
 396        for output in p4CmdList(" ".join(args)):
 397            print 'Element: %d' % j
 398            j += 1
 399            print output
 400        return True
 401
 402class P4RollBack(Command):
 403    def __init__(self):
 404        Command.__init__(self)
 405        self.options = [
 406            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408        ]
 409        self.description = "A tool to debug the multi-branch import. Don't use :)"
 410        self.verbose = False
 411        self.rollbackLocalBranches = False
 412
 413    def run(self, args):
 414        if len(args) != 1:
 415            return False
 416        maxChange = int(args[0])
 417
 418        if "p4ExitCode" in p4Cmd("changes -m 1"):
 419            die("Problems executing p4");
 420
 421        if self.rollbackLocalBranches:
 422            refPrefix = "refs/heads/"
 423            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424        else:
 425            refPrefix = "refs/remotes/"
 426            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428        for line in lines:
 429            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                line = line.strip()
 431                ref = refPrefix + line
 432                log = extractLogMessageFromGitCommit(ref)
 433                settings = extractSettingsGitLog(log)
 434
 435                depotPaths = settings['depot-paths']
 436                change = settings['change']
 437
 438                changed = False
 439
 440                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                           for p in depotPaths]))) == 0:
 442                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                    continue
 445
 446                while change and int(change) > maxChange:
 447                    changed = True
 448                    if self.verbose:
 449                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                    system("git update-ref %s \"%s^\"" % (ref, ref))
 451                    log = extractLogMessageFromGitCommit(ref)
 452                    settings =  extractSettingsGitLog(log)
 453
 454
 455                    depotPaths = settings['depot-paths']
 456                    change = settings['change']
 457
 458                if changed:
 459                    print "%s rewound to %s" % (ref, change)
 460
 461        return True
 462
 463class P4Submit(Command):
 464    def __init__(self):
 465        Command.__init__(self)
 466        self.options = [
 467                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 468                optparse.make_option("--origin", dest="origin"),
 469                optparse.make_option("-M", dest="detectRename", action="store_true"),
 470        ]
 471        self.description = "Submit changes from git to the perforce depot."
 472        self.usage += " [name of git branch to submit into perforce depot]"
 473        self.interactive = True
 474        self.origin = ""
 475        self.detectRename = False
 476        self.verbose = False
 477        self.isWindows = (platform.system() == "Windows")
 478
 479    def check(self):
 480        if len(p4CmdList("opened ...")) > 0:
 481            die("You have files opened with perforce! Close them before starting the sync.")
 482
 483    # replaces everything between 'Description:' and the next P4 submit template field with the
 484    # commit message
 485    def prepareLogMessage(self, template, message):
 486        result = ""
 487
 488        inDescriptionSection = False
 489
 490        for line in template.split("\n"):
 491            if line.startswith("#"):
 492                result += line + "\n"
 493                continue
 494
 495            if inDescriptionSection:
 496                if line.startswith("Files:"):
 497                    inDescriptionSection = False
 498                else:
 499                    continue
 500            else:
 501                if line.startswith("Description:"):
 502                    inDescriptionSection = True
 503                    line += "\n"
 504                    for messageLine in message.split("\n"):
 505                        line += "\t" + messageLine + "\n"
 506
 507            result += line + "\n"
 508
 509        return result
 510
 511    def prepareSubmitTemplate(self):
 512        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 513        template = ""
 514        inFilesSection = False
 515        for line in read_pipe_lines("p4 change -o"):
 516            if line.endswith("\r\n"):
 517                line = line[:-2] + "\n"
 518            if inFilesSection:
 519                if line.startswith("\t"):
 520                    # path starts and ends with a tab
 521                    path = line[1:]
 522                    lastTab = path.rfind("\t")
 523                    if lastTab != -1:
 524                        path = path[:lastTab]
 525                        if not path.startswith(self.depotPath):
 526                            continue
 527                else:
 528                    inFilesSection = False
 529            else:
 530                if line.startswith("Files:"):
 531                    inFilesSection = True
 532
 533            template += line
 534
 535        return template
 536
 537    def applyCommit(self, id):
 538        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 539        diffOpts = ("", "-M")[self.detectRename]
 540        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 541        filesToAdd = set()
 542        filesToDelete = set()
 543        editedFiles = set()
 544        filesToChangeExecBit = {}
 545        for line in diff:
 546            diff = parseDiffTreeEntry(line)
 547            modifier = diff['status']
 548            path = diff['src']
 549            if modifier == "M":
 550                system("p4 edit \"%s\"" % path)
 551                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 552                    filesToChangeExecBit[path] = diff['dst_mode']
 553                editedFiles.add(path)
 554            elif modifier == "A":
 555                filesToAdd.add(path)
 556                filesToChangeExecBit[path] = diff['dst_mode']
 557                if path in filesToDelete:
 558                    filesToDelete.remove(path)
 559            elif modifier == "D":
 560                filesToDelete.add(path)
 561                if path in filesToAdd:
 562                    filesToAdd.remove(path)
 563            elif modifier == "R":
 564                src, dest = diff['src'], diff['dst']
 565                system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 566                system("p4 edit \"%s\"" % (dest))
 567                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 568                    filesToChangeExecBit[dest] = diff['dst_mode']
 569                os.unlink(dest)
 570                editedFiles.add(dest)
 571                filesToDelete.add(src)
 572            else:
 573                die("unknown modifier %s for %s" % (modifier, path))
 574
 575        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 576        patchcmd = diffcmd + " | git apply "
 577        tryPatchCmd = patchcmd + "--check -"
 578        applyPatchCmd = patchcmd + "--check --apply -"
 579
 580        if os.system(tryPatchCmd) != 0:
 581            print "Unfortunately applying the change failed!"
 582            print "What do you want to do?"
 583            response = "x"
 584            while response != "s" and response != "a" and response != "w":
 585                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 586                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 587            if response == "s":
 588                print "Skipping! Good luck with the next patches..."
 589                for f in editedFiles:
 590                    system("p4 revert \"%s\"" % f);
 591                for f in filesToAdd:
 592                    system("rm %s" %f)
 593                return
 594            elif response == "a":
 595                os.system(applyPatchCmd)
 596                if len(filesToAdd) > 0:
 597                    print "You may also want to call p4 add on the following files:"
 598                    print " ".join(filesToAdd)
 599                if len(filesToDelete):
 600                    print "The following files should be scheduled for deletion with p4 delete:"
 601                    print " ".join(filesToDelete)
 602                die("Please resolve and submit the conflict manually and "
 603                    + "continue afterwards with git-p4 submit --continue")
 604            elif response == "w":
 605                system(diffcmd + " > patch.txt")
 606                print "Patch saved to patch.txt in %s !" % self.clientPath
 607                die("Please resolve and submit the conflict manually and "
 608                    "continue afterwards with git-p4 submit --continue")
 609
 610        system(applyPatchCmd)
 611
 612        for f in filesToAdd:
 613            system("p4 add \"%s\"" % f)
 614        for f in filesToDelete:
 615            system("p4 revert \"%s\"" % f)
 616            system("p4 delete \"%s\"" % f)
 617
 618        # Set/clear executable bits
 619        for f in filesToChangeExecBit.keys():
 620            mode = filesToChangeExecBit[f]
 621            setP4ExecBit(f, mode)
 622
 623        logMessage = extractLogMessageFromGitCommit(id)
 624        logMessage = logMessage.strip()
 625
 626        template = self.prepareSubmitTemplate()
 627
 628        if self.interactive:
 629            submitTemplate = self.prepareLogMessage(template, logMessage)
 630            if os.environ.has_key("P4DIFF"):
 631                del(os.environ["P4DIFF"])
 632            diff = read_pipe("p4 diff -du ...")
 633
 634            newdiff = ""
 635            for newFile in filesToAdd:
 636                newdiff += "==== new file ====\n"
 637                newdiff += "--- /dev/null\n"
 638                newdiff += "+++ %s\n" % newFile
 639                f = open(newFile, "r")
 640                for line in f.readlines():
 641                    newdiff += "+" + line
 642                f.close()
 643
 644            separatorLine = "######## everything below this line is just the diff #######\n"
 645
 646            [handle, fileName] = tempfile.mkstemp()
 647            tmpFile = os.fdopen(handle, "w+")
 648            if self.isWindows:
 649                submitTemplate = submitTemplate.replace("\n", "\r\n")
 650                separatorLine = separatorLine.replace("\n", "\r\n")
 651                newdiff = newdiff.replace("\n", "\r\n")
 652            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 653            tmpFile.close()
 654            defaultEditor = "vi"
 655            if platform.system() == "Windows":
 656                defaultEditor = "notepad"
 657            if os.environ.has_key("P4EDITOR"):
 658                editor = os.environ.get("P4EDITOR")
 659            else:
 660                editor = os.environ.get("EDITOR", defaultEditor);
 661            system(editor + " " + fileName)
 662            tmpFile = open(fileName, "rb")
 663            message = tmpFile.read()
 664            tmpFile.close()
 665            os.remove(fileName)
 666            submitTemplate = message[:message.index(separatorLine)]
 667            if self.isWindows:
 668                submitTemplate = submitTemplate.replace("\r\n", "\n")
 669
 670            write_pipe("p4 submit -i", submitTemplate)
 671        else:
 672            fileName = "submit.txt"
 673            file = open(fileName, "w+")
 674            file.write(self.prepareLogMessage(template, logMessage))
 675            file.close()
 676            print ("Perforce submit template written as %s. "
 677                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 678                   % (fileName, fileName))
 679
 680    def run(self, args):
 681        if len(args) == 0:
 682            self.master = currentGitBranch()
 683            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 684                die("Detecting current git branch failed!")
 685        elif len(args) == 1:
 686            self.master = args[0]
 687        else:
 688            return False
 689
 690        [upstream, settings] = findUpstreamBranchPoint()
 691        self.depotPath = settings['depot-paths'][0]
 692        if len(self.origin) == 0:
 693            self.origin = upstream
 694
 695        if self.verbose:
 696            print "Origin branch is " + self.origin
 697
 698        if len(self.depotPath) == 0:
 699            print "Internal error: cannot locate perforce depot path from existing branches"
 700            sys.exit(128)
 701
 702        self.clientPath = p4Where(self.depotPath)
 703
 704        if len(self.clientPath) == 0:
 705            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 706            sys.exit(128)
 707
 708        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 709        self.oldWorkingDirectory = os.getcwd()
 710
 711        os.chdir(self.clientPath)
 712        print "Syncronizing p4 checkout..."
 713        system("p4 sync ...")
 714
 715        self.check()
 716
 717        commits = []
 718        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 719            commits.append(line.strip())
 720        commits.reverse()
 721
 722        while len(commits) > 0:
 723            commit = commits[0]
 724            commits = commits[1:]
 725            self.applyCommit(commit)
 726            if not self.interactive:
 727                break
 728
 729        if len(commits) == 0:
 730            print "All changes applied!"
 731            os.chdir(self.oldWorkingDirectory)
 732
 733            sync = P4Sync()
 734            sync.run([])
 735
 736            rebase = P4Rebase()
 737            rebase.rebase()
 738
 739        return True
 740
 741class P4Sync(Command):
 742    def __init__(self):
 743        Command.__init__(self)
 744        self.options = [
 745                optparse.make_option("--branch", dest="branch"),
 746                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 747                optparse.make_option("--changesfile", dest="changesFile"),
 748                optparse.make_option("--silent", dest="silent", action="store_true"),
 749                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 750                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 751                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 752                                     help="Import into refs/heads/ , not refs/remotes"),
 753                optparse.make_option("--max-changes", dest="maxChanges"),
 754                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 755                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 756                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 757                                     help="Only sync files that are included in the Perforce Client Spec")
 758        ]
 759        self.description = """Imports from Perforce into a git repository.\n
 760    example:
 761    //depot/my/project/ -- to import the current head
 762    //depot/my/project/@all -- to import everything
 763    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 764
 765    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 766
 767        self.usage += " //depot/path[@revRange]"
 768        self.silent = False
 769        self.createdBranches = Set()
 770        self.committedChanges = Set()
 771        self.branch = ""
 772        self.detectBranches = False
 773        self.detectLabels = False
 774        self.changesFile = ""
 775        self.syncWithOrigin = True
 776        self.verbose = False
 777        self.importIntoRemotes = True
 778        self.maxChanges = ""
 779        self.isWindows = (platform.system() == "Windows")
 780        self.keepRepoPath = False
 781        self.depotPaths = None
 782        self.p4BranchesInGit = []
 783        self.cloneExclude = []
 784        self.useClientSpec = False
 785        self.clientSpecDirs = []
 786
 787        if gitConfig("git-p4.syncFromOrigin") == "false":
 788            self.syncWithOrigin = False
 789
 790    def extractFilesFromCommit(self, commit):
 791        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 792                             for path in self.cloneExclude]
 793        files = []
 794        fnum = 0
 795        while commit.has_key("depotFile%s" % fnum):
 796            path =  commit["depotFile%s" % fnum]
 797
 798            if [p for p in self.cloneExclude
 799                if path.startswith (p)]:
 800                found = False
 801            else:
 802                found = [p for p in self.depotPaths
 803                         if path.startswith (p)]
 804            if not found:
 805                fnum = fnum + 1
 806                continue
 807
 808            file = {}
 809            file["path"] = path
 810            file["rev"] = commit["rev%s" % fnum]
 811            file["action"] = commit["action%s" % fnum]
 812            file["type"] = commit["type%s" % fnum]
 813            files.append(file)
 814            fnum = fnum + 1
 815        return files
 816
 817    def stripRepoPath(self, path, prefixes):
 818        if self.keepRepoPath:
 819            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 820
 821        for p in prefixes:
 822            if path.startswith(p):
 823                path = path[len(p):]
 824
 825        return path
 826
 827    def splitFilesIntoBranches(self, commit):
 828        branches = {}
 829        fnum = 0
 830        while commit.has_key("depotFile%s" % fnum):
 831            path =  commit["depotFile%s" % fnum]
 832            found = [p for p in self.depotPaths
 833                     if path.startswith (p)]
 834            if not found:
 835                fnum = fnum + 1
 836                continue
 837
 838            file = {}
 839            file["path"] = path
 840            file["rev"] = commit["rev%s" % fnum]
 841            file["action"] = commit["action%s" % fnum]
 842            file["type"] = commit["type%s" % fnum]
 843            fnum = fnum + 1
 844
 845            relPath = self.stripRepoPath(path, self.depotPaths)
 846
 847            for branch in self.knownBranches.keys():
 848
 849                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 850                if relPath.startswith(branch + "/"):
 851                    if branch not in branches:
 852                        branches[branch] = []
 853                    branches[branch].append(file)
 854                    break
 855
 856        return branches
 857
 858    ## Should move this out, doesn't use SELF.
 859    def readP4Files(self, files):
 860        filesForCommit = []
 861        filesToRead = []
 862
 863        for f in files:
 864            includeFile = True
 865            for val in self.clientSpecDirs:
 866                if f['path'].startswith(val[0]):
 867                    if val[1] <= 0:
 868                        includeFile = False
 869                    break
 870
 871            if includeFile:
 872                filesForCommit.append(f)
 873                if f['action'] != 'delete':
 874                    filesToRead.append(f)
 875
 876        filedata = []
 877        if len(filesToRead) > 0:
 878            filedata = p4CmdList('-x - print',
 879                                 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 880                                                  for f in filesToRead]),
 881                                 stdin_mode='w+')
 882
 883            if "p4ExitCode" in filedata[0]:
 884                die("Problems executing p4. Error: [%d]."
 885                    % (filedata[0]['p4ExitCode']));
 886
 887        j = 0;
 888        contents = {}
 889        while j < len(filedata):
 890            stat = filedata[j]
 891            j += 1
 892            text = [];
 893            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 894                text.append(filedata[j]['data'])
 895                j += 1
 896            text = ''.join(text)
 897
 898            if not stat.has_key('depotFile'):
 899                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 900                continue
 901
 902            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 903                text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 904            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 905                text = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 906
 907            contents[stat['depotFile']] = text
 908
 909        for f in filesForCommit:
 910            path = f['path']
 911            if contents.has_key(path):
 912                f['data'] = contents[path]
 913
 914        return filesForCommit
 915
 916    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 917        epoch = details["time"]
 918        author = details["user"]
 919
 920        if self.verbose:
 921            print "commit into %s" % branch
 922
 923        # start with reading files; if that fails, we should not
 924        # create a commit.
 925        new_files = []
 926        for f in files:
 927            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 928                new_files.append (f)
 929            else:
 930                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 931        files = self.readP4Files(new_files)
 932
 933        self.gitStream.write("commit %s\n" % branch)
 934#        gitStream.write("mark :%s\n" % details["change"])
 935        self.committedChanges.add(int(details["change"]))
 936        committer = ""
 937        if author not in self.users:
 938            self.getUserMapFromPerforceServer()
 939        if author in self.users:
 940            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 941        else:
 942            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 943
 944        self.gitStream.write("committer %s\n" % committer)
 945
 946        self.gitStream.write("data <<EOT\n")
 947        self.gitStream.write(details["desc"])
 948        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 949                             % (','.join (branchPrefixes), details["change"]))
 950        if len(details['options']) > 0:
 951            self.gitStream.write(": options = %s" % details['options'])
 952        self.gitStream.write("]\nEOT\n\n")
 953
 954        if len(parent) > 0:
 955            if self.verbose:
 956                print "parent %s" % parent
 957            self.gitStream.write("from %s\n" % parent)
 958
 959        for file in files:
 960            if file["type"] == "apple":
 961                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 962                continue
 963
 964            relPath = self.stripRepoPath(file['path'], branchPrefixes)
 965            if file["action"] == "delete":
 966                self.gitStream.write("D %s\n" % relPath)
 967            else:
 968                data = file['data']
 969
 970                mode = "644"
 971                if isP4Exec(file["type"]):
 972                    mode = "755"
 973                elif file["type"] == "symlink":
 974                    mode = "120000"
 975                    # p4 print on a symlink contains "target\n", so strip it off
 976                    data = data[:-1]
 977
 978                if self.isWindows and file["type"].endswith("text"):
 979                    data = data.replace("\r\n", "\n")
 980
 981                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 982                self.gitStream.write("data %s\n" % len(data))
 983                self.gitStream.write(data)
 984                self.gitStream.write("\n")
 985
 986        self.gitStream.write("\n")
 987
 988        change = int(details["change"])
 989
 990        if self.labels.has_key(change):
 991            label = self.labels[change]
 992            labelDetails = label[0]
 993            labelRevisions = label[1]
 994            if self.verbose:
 995                print "Change %s is labelled %s" % (change, labelDetails)
 996
 997            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
 998                                                    for p in branchPrefixes]))
 999
1000            if len(files) == len(labelRevisions):
1001
1002                cleanedFiles = {}
1003                for info in files:
1004                    if info["action"] == "delete":
1005                        continue
1006                    cleanedFiles[info["depotFile"]] = info["rev"]
1007
1008                if cleanedFiles == labelRevisions:
1009                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1010                    self.gitStream.write("from %s\n" % branch)
1011
1012                    owner = labelDetails["Owner"]
1013                    tagger = ""
1014                    if author in self.users:
1015                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1016                    else:
1017                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1018                    self.gitStream.write("tagger %s\n" % tagger)
1019                    self.gitStream.write("data <<EOT\n")
1020                    self.gitStream.write(labelDetails["Description"])
1021                    self.gitStream.write("EOT\n\n")
1022
1023                else:
1024                    if not self.silent:
1025                        print ("Tag %s does not match with change %s: files do not match."
1026                               % (labelDetails["label"], change))
1027
1028            else:
1029                if not self.silent:
1030                    print ("Tag %s does not match with change %s: file count is different."
1031                           % (labelDetails["label"], change))
1032
1033    def getUserCacheFilename(self):
1034        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1035        return home + "/.gitp4-usercache.txt"
1036
1037    def getUserMapFromPerforceServer(self):
1038        if self.userMapFromPerforceServer:
1039            return
1040        self.users = {}
1041
1042        for output in p4CmdList("users"):
1043            if not output.has_key("User"):
1044                continue
1045            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1046
1047
1048        s = ''
1049        for (key, val) in self.users.items():
1050            s += "%s\t%s\n" % (key, val)
1051
1052        open(self.getUserCacheFilename(), "wb").write(s)
1053        self.userMapFromPerforceServer = True
1054
1055    def loadUserMapFromCache(self):
1056        self.users = {}
1057        self.userMapFromPerforceServer = False
1058        try:
1059            cache = open(self.getUserCacheFilename(), "rb")
1060            lines = cache.readlines()
1061            cache.close()
1062            for line in lines:
1063                entry = line.strip().split("\t")
1064                self.users[entry[0]] = entry[1]
1065        except IOError:
1066            self.getUserMapFromPerforceServer()
1067
1068    def getLabels(self):
1069        self.labels = {}
1070
1071        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1072        if len(l) > 0 and not self.silent:
1073            print "Finding files belonging to labels in %s" % `self.depotPaths`
1074
1075        for output in l:
1076            label = output["label"]
1077            revisions = {}
1078            newestChange = 0
1079            if self.verbose:
1080                print "Querying files for label %s" % label
1081            for file in p4CmdList("files "
1082                                  +  ' '.join (["%s...@%s" % (p, label)
1083                                                for p in self.depotPaths])):
1084                revisions[file["depotFile"]] = file["rev"]
1085                change = int(file["change"])
1086                if change > newestChange:
1087                    newestChange = change
1088
1089            self.labels[newestChange] = [output, revisions]
1090
1091        if self.verbose:
1092            print "Label changes: %s" % self.labels.keys()
1093
1094    def guessProjectName(self):
1095        for p in self.depotPaths:
1096            if p.endswith("/"):
1097                p = p[:-1]
1098            p = p[p.strip().rfind("/") + 1:]
1099            if not p.endswith("/"):
1100               p += "/"
1101            return p
1102
1103    def getBranchMapping(self):
1104        lostAndFoundBranches = set()
1105
1106        for info in p4CmdList("branches"):
1107            details = p4Cmd("branch -o %s" % info["branch"])
1108            viewIdx = 0
1109            while details.has_key("View%s" % viewIdx):
1110                paths = details["View%s" % viewIdx].split(" ")
1111                viewIdx = viewIdx + 1
1112                # require standard //depot/foo/... //depot/bar/... mapping
1113                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1114                    continue
1115                source = paths[0]
1116                destination = paths[1]
1117                ## HACK
1118                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1119                    source = source[len(self.depotPaths[0]):-4]
1120                    destination = destination[len(self.depotPaths[0]):-4]
1121
1122                    if destination in self.knownBranches:
1123                        if not self.silent:
1124                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1125                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1126                        continue
1127
1128                    self.knownBranches[destination] = source
1129
1130                    lostAndFoundBranches.discard(destination)
1131
1132                    if source not in self.knownBranches:
1133                        lostAndFoundBranches.add(source)
1134
1135
1136        for branch in lostAndFoundBranches:
1137            self.knownBranches[branch] = branch
1138
1139    def getBranchMappingFromGitBranches(self):
1140        branches = p4BranchesInGit(self.importIntoRemotes)
1141        for branch in branches.keys():
1142            if branch == "master":
1143                branch = "main"
1144            else:
1145                branch = branch[len(self.projectName):]
1146            self.knownBranches[branch] = branch
1147
1148    def listExistingP4GitBranches(self):
1149        # branches holds mapping from name to commit
1150        branches = p4BranchesInGit(self.importIntoRemotes)
1151        self.p4BranchesInGit = branches.keys()
1152        for branch in branches.keys():
1153            self.initialParents[self.refPrefix + branch] = branches[branch]
1154
1155    def updateOptionDict(self, d):
1156        option_keys = {}
1157        if self.keepRepoPath:
1158            option_keys['keepRepoPath'] = 1
1159
1160        d["options"] = ' '.join(sorted(option_keys.keys()))
1161
1162    def readOptions(self, d):
1163        self.keepRepoPath = (d.has_key('options')
1164                             and ('keepRepoPath' in d['options']))
1165
1166    def gitRefForBranch(self, branch):
1167        if branch == "main":
1168            return self.refPrefix + "master"
1169
1170        if len(branch) <= 0:
1171            return branch
1172
1173        return self.refPrefix + self.projectName + branch
1174
1175    def gitCommitByP4Change(self, ref, change):
1176        if self.verbose:
1177            print "looking in ref " + ref + " for change %s using bisect..." % change
1178
1179        earliestCommit = ""
1180        latestCommit = parseRevision(ref)
1181
1182        while True:
1183            if self.verbose:
1184                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1185            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1186            if len(next) == 0:
1187                if self.verbose:
1188                    print "argh"
1189                return ""
1190            log = extractLogMessageFromGitCommit(next)
1191            settings = extractSettingsGitLog(log)
1192            currentChange = int(settings['change'])
1193            if self.verbose:
1194                print "current change %s" % currentChange
1195
1196            if currentChange == change:
1197                if self.verbose:
1198                    print "found %s" % next
1199                return next
1200
1201            if currentChange < change:
1202                earliestCommit = "^%s" % next
1203            else:
1204                latestCommit = "%s" % next
1205
1206        return ""
1207
1208    def importNewBranch(self, branch, maxChange):
1209        # make fast-import flush all changes to disk and update the refs using the checkpoint
1210        # command so that we can try to find the branch parent in the git history
1211        self.gitStream.write("checkpoint\n\n");
1212        self.gitStream.flush();
1213        branchPrefix = self.depotPaths[0] + branch + "/"
1214        range = "@1,%s" % maxChange
1215        #print "prefix" + branchPrefix
1216        changes = p4ChangesForPaths([branchPrefix], range)
1217        if len(changes) <= 0:
1218            return False
1219        firstChange = changes[0]
1220        #print "first change in branch: %s" % firstChange
1221        sourceBranch = self.knownBranches[branch]
1222        sourceDepotPath = self.depotPaths[0] + sourceBranch
1223        sourceRef = self.gitRefForBranch(sourceBranch)
1224        #print "source " + sourceBranch
1225
1226        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1227        #print "branch parent: %s" % branchParentChange
1228        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1229        if len(gitParent) > 0:
1230            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1231            #print "parent git commit: %s" % gitParent
1232
1233        self.importChanges(changes)
1234        return True
1235
1236    def importChanges(self, changes):
1237        cnt = 1
1238        for change in changes:
1239            description = p4Cmd("describe %s" % change)
1240            self.updateOptionDict(description)
1241
1242            if not self.silent:
1243                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1244                sys.stdout.flush()
1245            cnt = cnt + 1
1246
1247            try:
1248                if self.detectBranches:
1249                    branches = self.splitFilesIntoBranches(description)
1250                    for branch in branches.keys():
1251                        ## HACK  --hwn
1252                        branchPrefix = self.depotPaths[0] + branch + "/"
1253
1254                        parent = ""
1255
1256                        filesForCommit = branches[branch]
1257
1258                        if self.verbose:
1259                            print "branch is %s" % branch
1260
1261                        self.updatedBranches.add(branch)
1262
1263                        if branch not in self.createdBranches:
1264                            self.createdBranches.add(branch)
1265                            parent = self.knownBranches[branch]
1266                            if parent == branch:
1267                                parent = ""
1268                            else:
1269                                fullBranch = self.projectName + branch
1270                                if fullBranch not in self.p4BranchesInGit:
1271                                    if not self.silent:
1272                                        print("\n    Importing new branch %s" % fullBranch);
1273                                    if self.importNewBranch(branch, change - 1):
1274                                        parent = ""
1275                                        self.p4BranchesInGit.append(fullBranch)
1276                                    if not self.silent:
1277                                        print("\n    Resuming with change %s" % change);
1278
1279                                if self.verbose:
1280                                    print "parent determined through known branches: %s" % parent
1281
1282                        branch = self.gitRefForBranch(branch)
1283                        parent = self.gitRefForBranch(parent)
1284
1285                        if self.verbose:
1286                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1287
1288                        if len(parent) == 0 and branch in self.initialParents:
1289                            parent = self.initialParents[branch]
1290                            del self.initialParents[branch]
1291
1292                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1293                else:
1294                    files = self.extractFilesFromCommit(description)
1295                    self.commit(description, files, self.branch, self.depotPaths,
1296                                self.initialParent)
1297                    self.initialParent = ""
1298            except IOError:
1299                print self.gitError.read()
1300                sys.exit(1)
1301
1302    def importHeadRevision(self, revision):
1303        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1304
1305        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1306        details["desc"] = ("Initial import of %s from the state at revision %s"
1307                           % (' '.join(self.depotPaths), revision))
1308        details["change"] = revision
1309        newestRevision = 0
1310
1311        fileCnt = 0
1312        for info in p4CmdList("files "
1313                              +  ' '.join(["%s...%s"
1314                                           % (p, revision)
1315                                           for p in self.depotPaths])):
1316
1317            if info['code'] == 'error':
1318                sys.stderr.write("p4 returned an error: %s\n"
1319                                 % info['data'])
1320                sys.exit(1)
1321
1322
1323            change = int(info["change"])
1324            if change > newestRevision:
1325                newestRevision = change
1326
1327            if info["action"] == "delete":
1328                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1329                #fileCnt = fileCnt + 1
1330                continue
1331
1332            for prop in ["depotFile", "rev", "action", "type" ]:
1333                details["%s%s" % (prop, fileCnt)] = info[prop]
1334
1335            fileCnt = fileCnt + 1
1336
1337        details["change"] = newestRevision
1338        self.updateOptionDict(details)
1339        try:
1340            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1341        except IOError:
1342            print "IO error with git fast-import. Is your git version recent enough?"
1343            print self.gitError.read()
1344
1345
1346    def getClientSpec(self):
1347        specList = p4CmdList( "client -o" )
1348        temp = {}
1349        for entry in specList:
1350            for k,v in entry.iteritems():
1351                if k.startswith("View"):
1352                    if v.startswith('"'):
1353                        start = 1
1354                    else:
1355                        start = 0
1356                    index = v.find("...")
1357                    v = v[start:index]
1358                    if v.startswith("-"):
1359                        v = v[1:]
1360                        temp[v] = -len(v)
1361                    else:
1362                        temp[v] = len(v)
1363        self.clientSpecDirs = temp.items()
1364        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1365
1366    def run(self, args):
1367        self.depotPaths = []
1368        self.changeRange = ""
1369        self.initialParent = ""
1370        self.previousDepotPaths = []
1371
1372        # map from branch depot path to parent branch
1373        self.knownBranches = {}
1374        self.initialParents = {}
1375        self.hasOrigin = originP4BranchesExist()
1376        if not self.syncWithOrigin:
1377            self.hasOrigin = False
1378
1379        if self.importIntoRemotes:
1380            self.refPrefix = "refs/remotes/p4/"
1381        else:
1382            self.refPrefix = "refs/heads/p4/"
1383
1384        if self.syncWithOrigin and self.hasOrigin:
1385            if not self.silent:
1386                print "Syncing with origin first by calling git fetch origin"
1387            system("git fetch origin")
1388
1389        if len(self.branch) == 0:
1390            self.branch = self.refPrefix + "master"
1391            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1392                system("git update-ref %s refs/heads/p4" % self.branch)
1393                system("git branch -D p4");
1394            # create it /after/ importing, when master exists
1395            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1396                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1397
1398        if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1399            self.getClientSpec()
1400
1401        # TODO: should always look at previous commits,
1402        # merge with previous imports, if possible.
1403        if args == []:
1404            if self.hasOrigin:
1405                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1406            self.listExistingP4GitBranches()
1407
1408            if len(self.p4BranchesInGit) > 1:
1409                if not self.silent:
1410                    print "Importing from/into multiple branches"
1411                self.detectBranches = True
1412
1413            if self.verbose:
1414                print "branches: %s" % self.p4BranchesInGit
1415
1416            p4Change = 0
1417            for branch in self.p4BranchesInGit:
1418                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1419
1420                settings = extractSettingsGitLog(logMsg)
1421
1422                self.readOptions(settings)
1423                if (settings.has_key('depot-paths')
1424                    and settings.has_key ('change')):
1425                    change = int(settings['change']) + 1
1426                    p4Change = max(p4Change, change)
1427
1428                    depotPaths = sorted(settings['depot-paths'])
1429                    if self.previousDepotPaths == []:
1430                        self.previousDepotPaths = depotPaths
1431                    else:
1432                        paths = []
1433                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1434                            for i in range(0, min(len(cur), len(prev))):
1435                                if cur[i] <> prev[i]:
1436                                    i = i - 1
1437                                    break
1438
1439                            paths.append (cur[:i + 1])
1440
1441                        self.previousDepotPaths = paths
1442
1443            if p4Change > 0:
1444                self.depotPaths = sorted(self.previousDepotPaths)
1445                self.changeRange = "@%s,#head" % p4Change
1446                if not self.detectBranches:
1447                    self.initialParent = parseRevision(self.branch)
1448                if not self.silent and not self.detectBranches:
1449                    print "Performing incremental import into %s git branch" % self.branch
1450
1451        if not self.branch.startswith("refs/"):
1452            self.branch = "refs/heads/" + self.branch
1453
1454        if len(args) == 0 and self.depotPaths:
1455            if not self.silent:
1456                print "Depot paths: %s" % ' '.join(self.depotPaths)
1457        else:
1458            if self.depotPaths and self.depotPaths != args:
1459                print ("previous import used depot path %s and now %s was specified. "
1460                       "This doesn't work!" % (' '.join (self.depotPaths),
1461                                               ' '.join (args)))
1462                sys.exit(1)
1463
1464            self.depotPaths = sorted(args)
1465
1466        revision = ""
1467        self.users = {}
1468
1469        newPaths = []
1470        for p in self.depotPaths:
1471            if p.find("@") != -1:
1472                atIdx = p.index("@")
1473                self.changeRange = p[atIdx:]
1474                if self.changeRange == "@all":
1475                    self.changeRange = ""
1476                elif ',' not in self.changeRange:
1477                    revision = self.changeRange
1478                    self.changeRange = ""
1479                p = p[:atIdx]
1480            elif p.find("#") != -1:
1481                hashIdx = p.index("#")
1482                revision = p[hashIdx:]
1483                p = p[:hashIdx]
1484            elif self.previousDepotPaths == []:
1485                revision = "#head"
1486
1487            p = re.sub ("\.\.\.$", "", p)
1488            if not p.endswith("/"):
1489                p += "/"
1490
1491            newPaths.append(p)
1492
1493        self.depotPaths = newPaths
1494
1495
1496        self.loadUserMapFromCache()
1497        self.labels = {}
1498        if self.detectLabels:
1499            self.getLabels();
1500
1501        if self.detectBranches:
1502            ## FIXME - what's a P4 projectName ?
1503            self.projectName = self.guessProjectName()
1504
1505            if self.hasOrigin:
1506                self.getBranchMappingFromGitBranches()
1507            else:
1508                self.getBranchMapping()
1509            if self.verbose:
1510                print "p4-git branches: %s" % self.p4BranchesInGit
1511                print "initial parents: %s" % self.initialParents
1512            for b in self.p4BranchesInGit:
1513                if b != "master":
1514
1515                    ## FIXME
1516                    b = b[len(self.projectName):]
1517                self.createdBranches.add(b)
1518
1519        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1520
1521        importProcess = subprocess.Popen(["git", "fast-import"],
1522                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1523                                         stderr=subprocess.PIPE);
1524        self.gitOutput = importProcess.stdout
1525        self.gitStream = importProcess.stdin
1526        self.gitError = importProcess.stderr
1527
1528        if revision:
1529            self.importHeadRevision(revision)
1530        else:
1531            changes = []
1532
1533            if len(self.changesFile) > 0:
1534                output = open(self.changesFile).readlines()
1535                changeSet = Set()
1536                for line in output:
1537                    changeSet.add(int(line))
1538
1539                for change in changeSet:
1540                    changes.append(change)
1541
1542                changes.sort()
1543            else:
1544                if self.verbose:
1545                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1546                                                              self.changeRange)
1547                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1548
1549                if len(self.maxChanges) > 0:
1550                    changes = changes[:min(int(self.maxChanges), len(changes))]
1551
1552            if len(changes) == 0:
1553                if not self.silent:
1554                    print "No changes to import!"
1555                return True
1556
1557            if not self.silent and not self.detectBranches:
1558                print "Import destination: %s" % self.branch
1559
1560            self.updatedBranches = set()
1561
1562            self.importChanges(changes)
1563
1564            if not self.silent:
1565                print ""
1566                if len(self.updatedBranches) > 0:
1567                    sys.stdout.write("Updated branches: ")
1568                    for b in self.updatedBranches:
1569                        sys.stdout.write("%s " % b)
1570                    sys.stdout.write("\n")
1571
1572        self.gitStream.close()
1573        if importProcess.wait() != 0:
1574            die("fast-import failed: %s" % self.gitError.read())
1575        self.gitOutput.close()
1576        self.gitError.close()
1577
1578        return True
1579
1580class P4Rebase(Command):
1581    def __init__(self):
1582        Command.__init__(self)
1583        self.options = [ ]
1584        self.description = ("Fetches the latest revision from perforce and "
1585                            + "rebases the current work (branch) against it")
1586        self.verbose = False
1587
1588    def run(self, args):
1589        sync = P4Sync()
1590        sync.run([])
1591
1592        return self.rebase()
1593
1594    def rebase(self):
1595        if os.system("git update-index --refresh") != 0:
1596            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1597        if len(read_pipe("git diff-index HEAD --")) > 0:
1598            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1599
1600        [upstream, settings] = findUpstreamBranchPoint()
1601        if len(upstream) == 0:
1602            die("Cannot find upstream branchpoint for rebase")
1603
1604        # the branchpoint may be p4/foo~3, so strip off the parent
1605        upstream = re.sub("~[0-9]+$", "", upstream)
1606
1607        print "Rebasing the current branch onto %s" % upstream
1608        oldHead = read_pipe("git rev-parse HEAD").strip()
1609        system("git rebase %s" % upstream)
1610        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1611        return True
1612
1613class P4Clone(P4Sync):
1614    def __init__(self):
1615        P4Sync.__init__(self)
1616        self.description = "Creates a new git repository and imports from Perforce into it"
1617        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1618        self.options += [
1619            optparse.make_option("--destination", dest="cloneDestination",
1620                                 action='store', default=None,
1621                                 help="where to leave result of the clone"),
1622            optparse.make_option("-/", dest="cloneExclude",
1623                                 action="append", type="string",
1624                                 help="exclude depot path")
1625        ]
1626        self.cloneDestination = None
1627        self.needsGit = False
1628
1629    # This is required for the "append" cloneExclude action
1630    def ensure_value(self, attr, value):
1631        if not hasattr(self, attr) or getattr(self, attr) is None:
1632            setattr(self, attr, value)
1633        return getattr(self, attr)
1634
1635    def defaultDestination(self, args):
1636        ## TODO: use common prefix of args?
1637        depotPath = args[0]
1638        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1639        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1640        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1641        depotDir = re.sub(r"/$", "", depotDir)
1642        return os.path.split(depotDir)[1]
1643
1644    def run(self, args):
1645        if len(args) < 1:
1646            return False
1647
1648        if self.keepRepoPath and not self.cloneDestination:
1649            sys.stderr.write("Must specify destination for --keep-path\n")
1650            sys.exit(1)
1651
1652        depotPaths = args
1653
1654        if not self.cloneDestination and len(depotPaths) > 1:
1655            self.cloneDestination = depotPaths[-1]
1656            depotPaths = depotPaths[:-1]
1657
1658        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1659        for p in depotPaths:
1660            if not p.startswith("//"):
1661                return False
1662
1663        if not self.cloneDestination:
1664            self.cloneDestination = self.defaultDestination(args)
1665
1666        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1667        if not os.path.exists(self.cloneDestination):
1668            os.makedirs(self.cloneDestination)
1669        os.chdir(self.cloneDestination)
1670        system("git init")
1671        self.gitdir = os.getcwd() + "/.git"
1672        if not P4Sync.run(self, depotPaths):
1673            return False
1674        if self.branch != "master":
1675            if gitBranchExists("refs/remotes/p4/master"):
1676                system("git branch master refs/remotes/p4/master")
1677                system("git checkout -f")
1678            else:
1679                print "Could not detect main branch. No checkout/master branch created."
1680
1681        return True
1682
1683class P4Branches(Command):
1684    def __init__(self):
1685        Command.__init__(self)
1686        self.options = [ ]
1687        self.description = ("Shows the git branches that hold imports and their "
1688                            + "corresponding perforce depot paths")
1689        self.verbose = False
1690
1691    def run(self, args):
1692        if originP4BranchesExist():
1693            createOrUpdateBranchesFromOrigin()
1694
1695        cmdline = "git rev-parse --symbolic "
1696        cmdline += " --remotes"
1697
1698        for line in read_pipe_lines(cmdline):
1699            line = line.strip()
1700
1701            if not line.startswith('p4/') or line == "p4/HEAD":
1702                continue
1703            branch = line
1704
1705            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1706            settings = extractSettingsGitLog(log)
1707
1708            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1709        return True
1710
1711class HelpFormatter(optparse.IndentedHelpFormatter):
1712    def __init__(self):
1713        optparse.IndentedHelpFormatter.__init__(self)
1714
1715    def format_description(self, description):
1716        if description:
1717            return description + "\n"
1718        else:
1719            return ""
1720
1721def printUsage(commands):
1722    print "usage: %s <command> [options]" % sys.argv[0]
1723    print ""
1724    print "valid commands: %s" % ", ".join(commands)
1725    print ""
1726    print "Try %s <command> --help for command specific help." % sys.argv[0]
1727    print ""
1728
1729commands = {
1730    "debug" : P4Debug,
1731    "submit" : P4Submit,
1732    "commit" : P4Submit,
1733    "sync" : P4Sync,
1734    "rebase" : P4Rebase,
1735    "clone" : P4Clone,
1736    "rollback" : P4RollBack,
1737    "branches" : P4Branches
1738}
1739
1740
1741def main():
1742    if len(sys.argv[1:]) == 0:
1743        printUsage(commands.keys())
1744        sys.exit(2)
1745
1746    cmd = ""
1747    cmdName = sys.argv[1]
1748    try:
1749        klass = commands[cmdName]
1750        cmd = klass()
1751    except KeyError:
1752        print "unknown command %s" % cmdName
1753        print ""
1754        printUsage(commands.keys())
1755        sys.exit(2)
1756
1757    options = cmd.options
1758    cmd.gitdir = os.environ.get("GIT_DIR", None)
1759
1760    args = sys.argv[2:]
1761
1762    if len(options) > 0:
1763        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1764
1765        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1766                                       options,
1767                                       description = cmd.description,
1768                                       formatter = HelpFormatter())
1769
1770        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1771    global verbose
1772    verbose = cmd.verbose
1773    if cmd.needsGit:
1774        if cmd.gitdir == None:
1775            cmd.gitdir = os.path.abspath(".git")
1776            if not isValidGitDir(cmd.gitdir):
1777                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1778                if os.path.exists(cmd.gitdir):
1779                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1780                    if len(cdup) > 0:
1781                        os.chdir(cdup);
1782
1783        if not isValidGitDir(cmd.gitdir):
1784            if isValidGitDir(cmd.gitdir + "/.git"):
1785                cmd.gitdir += "/.git"
1786            else:
1787                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1788
1789        os.environ["GIT_DIR"] = cmd.gitdir
1790
1791    if not cmd.run(args):
1792        parser.print_help()
1793
1794
1795if __name__ == '__main__':
1796    main()