Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'maint' (a0653d5)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19def die(msg):
  20    if verbose:
  21        raise Exception(msg)
  22    else:
  23        sys.stderr.write(msg + "\n")
  24        sys.exit(1)
  25
  26def write_pipe(c, str):
  27    if verbose:
  28        sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30    pipe = os.popen(c, 'w')
  31    val = pipe.write(str)
  32    if pipe.close():
  33        die('Command failed: %s' % c)
  34
  35    return val
  36
  37def read_pipe(c, ignore_error=False):
  38    if verbose:
  39        sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41    pipe = os.popen(c, 'rb')
  42    val = pipe.read()
  43    if pipe.close() and not ignore_error:
  44        die('Command failed: %s' % c)
  45
  46    return val
  47
  48
  49def read_pipe_lines(c):
  50    if verbose:
  51        sys.stderr.write('Reading pipe: %s\n' % c)
  52    ## todo: check return status
  53    pipe = os.popen(c, 'rb')
  54    val = pipe.readlines()
  55    if pipe.close():
  56        die('Command failed: %s' % c)
  57
  58    return val
  59
  60def system(cmd):
  61    if verbose:
  62        sys.stderr.write("executing %s\n" % cmd)
  63    if os.system(cmd) != 0:
  64        die("command failed: %s" % cmd)
  65
  66def isP4Exec(kind):
  67    """Determine if a Perforce 'kind' should have execute permission
  68
  69    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70    or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71    a plus sign, it is also executable"""
  72    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74def setP4ExecBit(file, mode):
  75    # Reopens an already open file and changes the execute bit to match
  76    # the execute bit setting in the passed in mode.
  77
  78    p4Type = "+x"
  79
  80    if not isModeExec(mode):
  81        p4Type = getP4OpenedType(file)
  82        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84        if p4Type[-1] == "+":
  85            p4Type = p4Type[0:-1]
  86
  87    system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89def getP4OpenedType(file):
  90    # Returns the perforce file type for the given file.
  91
  92    result = read_pipe("p4 opened %s" % file)
  93    match = re.match(".*\((.+)\)\r?$", result)
  94    if match:
  95        return match.group(1)
  96    else:
  97        die("Could not determine file type for %s (result: '%s')" % (file, result))
  98
  99def diffTreePattern():
 100    # This is a simple generator for the diff tree regex pattern. This could be
 101    # a class variable if this and parseDiffTreeEntry were a part of a class.
 102    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103    while True:
 104        yield pattern
 105
 106def parseDiffTreeEntry(entry):
 107    """Parses a single diff tree entry into its component elements.
 108
 109    See git-diff-tree(1) manpage for details about the format of the diff
 110    output. This method returns a dictionary with the following elements:
 111
 112    src_mode - The mode of the source file
 113    dst_mode - The mode of the destination file
 114    src_sha1 - The sha1 for the source file
 115    dst_sha1 - The sha1 fr the destination file
 116    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117    status_score - The score for the status (applicable for 'C' and 'R'
 118                   statuses). This is None if there is no score.
 119    src - The path for the source file.
 120    dst - The path for the destination file. This is only present for
 121          copy or renames. If it is not present, this is None.
 122
 123    If the pattern is not matched, None is returned."""
 124
 125    match = diffTreePattern().next().match(entry)
 126    if match:
 127        return {
 128            'src_mode': match.group(1),
 129            'dst_mode': match.group(2),
 130            'src_sha1': match.group(3),
 131            'dst_sha1': match.group(4),
 132            'status': match.group(5),
 133            'status_score': match.group(6),
 134            'src': match.group(7),
 135            'dst': match.group(10)
 136        }
 137    return None
 138
 139def isModeExec(mode):
 140    # Returns True if the given git mode represents an executable file,
 141    # otherwise False.
 142    return mode[-3:] == "755"
 143
 144def isModeExecChanged(src_mode, dst_mode):
 145    return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148    cmd = "p4 -G %s" % cmd
 149    if verbose:
 150        sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152    # Use a temporary file to avoid deadlocks without
 153    # subprocess.communicate(), which would put another copy
 154    # of stdout into memory.
 155    stdin_file = None
 156    if stdin is not None:
 157        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158        stdin_file.write(stdin)
 159        stdin_file.flush()
 160        stdin_file.seek(0)
 161
 162    p4 = subprocess.Popen(cmd, shell=True,
 163                          stdin=stdin_file,
 164                          stdout=subprocess.PIPE)
 165
 166    result = []
 167    try:
 168        while True:
 169            entry = marshal.load(p4.stdout)
 170            result.append(entry)
 171    except EOFError:
 172        pass
 173    exitCode = p4.wait()
 174    if exitCode != 0:
 175        entry = {}
 176        entry["p4ExitCode"] = exitCode
 177        result.append(entry)
 178
 179    return result
 180
 181def p4Cmd(cmd):
 182    list = p4CmdList(cmd)
 183    result = {}
 184    for entry in list:
 185        result.update(entry)
 186    return result;
 187
 188def p4Where(depotPath):
 189    if not depotPath.endswith("/"):
 190        depotPath += "/"
 191    output = p4Cmd("where %s..." % depotPath)
 192    if output["code"] == "error":
 193        return ""
 194    clientPath = ""
 195    if "path" in output:
 196        clientPath = output.get("path")
 197    elif "data" in output:
 198        data = output.get("data")
 199        lastSpace = data.rfind(" ")
 200        clientPath = data[lastSpace + 1:]
 201
 202    if clientPath.endswith("..."):
 203        clientPath = clientPath[:-3]
 204    return clientPath
 205
 206def currentGitBranch():
 207    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209def isValidGitDir(path):
 210    if (os.path.exists(path + "/HEAD")
 211        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212        return True;
 213    return False
 214
 215def parseRevision(ref):
 216    return read_pipe("git rev-parse %s" % ref).strip()
 217
 218def extractLogMessageFromGitCommit(commit):
 219    logMessage = ""
 220
 221    ## fixme: title is first line of commit, not 1st paragraph.
 222    foundTitle = False
 223    for log in read_pipe_lines("git cat-file commit %s" % commit):
 224       if not foundTitle:
 225           if len(log) == 1:
 226               foundTitle = True
 227           continue
 228
 229       logMessage += log
 230    return logMessage
 231
 232def extractSettingsGitLog(log):
 233    values = {}
 234    for line in log.split("\n"):
 235        line = line.strip()
 236        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237        if not m:
 238            continue
 239
 240        assignments = m.group(1).split (':')
 241        for a in assignments:
 242            vals = a.split ('=')
 243            key = vals[0].strip()
 244            val = ('='.join (vals[1:])).strip()
 245            if val.endswith ('\"') and val.startswith('"'):
 246                val = val[1:-1]
 247
 248            values[key] = val
 249
 250    paths = values.get("depot-paths")
 251    if not paths:
 252        paths = values.get("depot-path")
 253    if paths:
 254        values['depot-paths'] = paths.split(',')
 255    return values
 256
 257def gitBranchExists(branch):
 258    proc = subprocess.Popen(["git", "rev-parse", branch],
 259                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260    return proc.wait() == 0;
 261
 262def gitConfig(key):
 263    return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265def p4BranchesInGit(branchesAreInRemotes = True):
 266    branches = {}
 267
 268    cmdline = "git rev-parse --symbolic "
 269    if branchesAreInRemotes:
 270        cmdline += " --remotes"
 271    else:
 272        cmdline += " --branches"
 273
 274    for line in read_pipe_lines(cmdline):
 275        line = line.strip()
 276
 277        ## only import to p4/
 278        if not line.startswith('p4/') or line == "p4/HEAD":
 279            continue
 280        branch = line
 281
 282        # strip off p4
 283        branch = re.sub ("^p4/", "", line)
 284
 285        branches[branch] = parseRevision(line)
 286    return branches
 287
 288def findUpstreamBranchPoint(head = "HEAD"):
 289    branches = p4BranchesInGit()
 290    # map from depot-path to branch name
 291    branchByDepotPath = {}
 292    for branch in branches.keys():
 293        tip = branches[branch]
 294        log = extractLogMessageFromGitCommit(tip)
 295        settings = extractSettingsGitLog(log)
 296        if settings.has_key("depot-paths"):
 297            paths = ",".join(settings["depot-paths"])
 298            branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300    settings = None
 301    parent = 0
 302    while parent < 65535:
 303        commit = head + "~%s" % parent
 304        log = extractLogMessageFromGitCommit(commit)
 305        settings = extractSettingsGitLog(log)
 306        if settings.has_key("depot-paths"):
 307            paths = ",".join(settings["depot-paths"])
 308            if branchByDepotPath.has_key(paths):
 309                return [branchByDepotPath[paths], settings]
 310
 311        parent = parent + 1
 312
 313    return ["", settings]
 314
 315def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316    if not silent:
 317        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318               % localRefPrefix)
 319
 320    originPrefix = "origin/p4/"
 321
 322    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323        line = line.strip()
 324        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325            continue
 326
 327        headName = line[len(originPrefix):]
 328        remoteHead = localRefPrefix + headName
 329        originHead = line
 330
 331        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332        if (not original.has_key('depot-paths')
 333            or not original.has_key('change')):
 334            continue
 335
 336        update = False
 337        if not gitBranchExists(remoteHead):
 338            if verbose:
 339                print "creating %s" % remoteHead
 340            update = True
 341        else:
 342            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343            if settings.has_key('change') > 0:
 344                if settings['depot-paths'] == original['depot-paths']:
 345                    originP4Change = int(original['change'])
 346                    p4Change = int(settings['change'])
 347                    if originP4Change > p4Change:
 348                        print ("%s (%s) is newer than %s (%s). "
 349                               "Updating p4 branch from origin."
 350                               % (originHead, originP4Change,
 351                                  remoteHead, p4Change))
 352                        update = True
 353                else:
 354                    print ("Ignoring: %s was imported from %s while "
 355                           "%s was imported from %s"
 356                           % (originHead, ','.join(original['depot-paths']),
 357                              remoteHead, ','.join(settings['depot-paths'])))
 358
 359        if update:
 360            system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362def originP4BranchesExist():
 363        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365def p4ChangesForPaths(depotPaths, changeRange):
 366    assert depotPaths
 367    output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                        for p in depotPaths]))
 369
 370    changes = []
 371    for line in output:
 372        changeNum = line.split(" ")[1]
 373        changes.append(int(changeNum))
 374
 375    changes.sort()
 376    return changes
 377
 378class Command:
 379    def __init__(self):
 380        self.usage = "usage: %prog [options]"
 381        self.needsGit = True
 382
 383class P4Debug(Command):
 384    def __init__(self):
 385        Command.__init__(self)
 386        self.options = [
 387            optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                 default=False),
 389            ]
 390        self.description = "A tool to debug the output of p4 -G."
 391        self.needsGit = False
 392        self.verbose = False
 393
 394    def run(self, args):
 395        j = 0
 396        for output in p4CmdList(" ".join(args)):
 397            print 'Element: %d' % j
 398            j += 1
 399            print output
 400        return True
 401
 402class P4RollBack(Command):
 403    def __init__(self):
 404        Command.__init__(self)
 405        self.options = [
 406            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408        ]
 409        self.description = "A tool to debug the multi-branch import. Don't use :)"
 410        self.verbose = False
 411        self.rollbackLocalBranches = False
 412
 413    def run(self, args):
 414        if len(args) != 1:
 415            return False
 416        maxChange = int(args[0])
 417
 418        if "p4ExitCode" in p4Cmd("changes -m 1"):
 419            die("Problems executing p4");
 420
 421        if self.rollbackLocalBranches:
 422            refPrefix = "refs/heads/"
 423            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424        else:
 425            refPrefix = "refs/remotes/"
 426            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428        for line in lines:
 429            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                line = line.strip()
 431                ref = refPrefix + line
 432                log = extractLogMessageFromGitCommit(ref)
 433                settings = extractSettingsGitLog(log)
 434
 435                depotPaths = settings['depot-paths']
 436                change = settings['change']
 437
 438                changed = False
 439
 440                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                           for p in depotPaths]))) == 0:
 442                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                    continue
 445
 446                while change and int(change) > maxChange:
 447                    changed = True
 448                    if self.verbose:
 449                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                    system("git update-ref %s \"%s^\"" % (ref, ref))
 451                    log = extractLogMessageFromGitCommit(ref)
 452                    settings =  extractSettingsGitLog(log)
 453
 454
 455                    depotPaths = settings['depot-paths']
 456                    change = settings['change']
 457
 458                if changed:
 459                    print "%s rewound to %s" % (ref, change)
 460
 461        return True
 462
 463class P4Submit(Command):
 464    def __init__(self):
 465        Command.__init__(self)
 466        self.options = [
 467                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 468                optparse.make_option("--origin", dest="origin"),
 469                optparse.make_option("-M", dest="detectRename", action="store_true"),
 470        ]
 471        self.description = "Submit changes from git to the perforce depot."
 472        self.usage += " [name of git branch to submit into perforce depot]"
 473        self.interactive = True
 474        self.origin = ""
 475        self.detectRename = False
 476        self.verbose = False
 477        self.isWindows = (platform.system() == "Windows")
 478
 479    def check(self):
 480        if len(p4CmdList("opened ...")) > 0:
 481            die("You have files opened with perforce! Close them before starting the sync.")
 482
 483    # replaces everything between 'Description:' and the next P4 submit template field with the
 484    # commit message
 485    def prepareLogMessage(self, template, message):
 486        result = ""
 487
 488        inDescriptionSection = False
 489
 490        for line in template.split("\n"):
 491            if line.startswith("#"):
 492                result += line + "\n"
 493                continue
 494
 495            if inDescriptionSection:
 496                if line.startswith("Files:"):
 497                    inDescriptionSection = False
 498                else:
 499                    continue
 500            else:
 501                if line.startswith("Description:"):
 502                    inDescriptionSection = True
 503                    line += "\n"
 504                    for messageLine in message.split("\n"):
 505                        line += "\t" + messageLine + "\n"
 506
 507            result += line + "\n"
 508
 509        return result
 510
 511    def prepareSubmitTemplate(self):
 512        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 513        template = ""
 514        inFilesSection = False
 515        for line in read_pipe_lines("p4 change -o"):
 516            if line.endswith("\r\n"):
 517                line = line[:-2] + "\n"
 518            if inFilesSection:
 519                if line.startswith("\t"):
 520                    # path starts and ends with a tab
 521                    path = line[1:]
 522                    lastTab = path.rfind("\t")
 523                    if lastTab != -1:
 524                        path = path[:lastTab]
 525                        if not path.startswith(self.depotPath):
 526                            continue
 527                else:
 528                    inFilesSection = False
 529            else:
 530                if line.startswith("Files:"):
 531                    inFilesSection = True
 532
 533            template += line
 534
 535        return template
 536
 537    def applyCommit(self, id):
 538        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 539        diffOpts = ("", "-M")[self.detectRename]
 540        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 541        filesToAdd = set()
 542        filesToDelete = set()
 543        editedFiles = set()
 544        filesToChangeExecBit = {}
 545        for line in diff:
 546            diff = parseDiffTreeEntry(line)
 547            modifier = diff['status']
 548            path = diff['src']
 549            if modifier == "M":
 550                system("p4 edit \"%s\"" % path)
 551                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 552                    filesToChangeExecBit[path] = diff['dst_mode']
 553                editedFiles.add(path)
 554            elif modifier == "A":
 555                filesToAdd.add(path)
 556                filesToChangeExecBit[path] = diff['dst_mode']
 557                if path in filesToDelete:
 558                    filesToDelete.remove(path)
 559            elif modifier == "D":
 560                filesToDelete.add(path)
 561                if path in filesToAdd:
 562                    filesToAdd.remove(path)
 563            elif modifier == "R":
 564                src, dest = diff['src'], diff['dst']
 565                system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 566                system("p4 edit \"%s\"" % (dest))
 567                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 568                    filesToChangeExecBit[dest] = diff['dst_mode']
 569                os.unlink(dest)
 570                editedFiles.add(dest)
 571                filesToDelete.add(src)
 572            else:
 573                die("unknown modifier %s for %s" % (modifier, path))
 574
 575        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 576        patchcmd = diffcmd + " | git apply "
 577        tryPatchCmd = patchcmd + "--check -"
 578        applyPatchCmd = patchcmd + "--check --apply -"
 579
 580        if os.system(tryPatchCmd) != 0:
 581            print "Unfortunately applying the change failed!"
 582            print "What do you want to do?"
 583            response = "x"
 584            while response != "s" and response != "a" and response != "w":
 585                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 586                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 587            if response == "s":
 588                print "Skipping! Good luck with the next patches..."
 589                for f in editedFiles:
 590                    system("p4 revert \"%s\"" % f);
 591                for f in filesToAdd:
 592                    system("rm %s" %f)
 593                return
 594            elif response == "a":
 595                os.system(applyPatchCmd)
 596                if len(filesToAdd) > 0:
 597                    print "You may also want to call p4 add on the following files:"
 598                    print " ".join(filesToAdd)
 599                if len(filesToDelete):
 600                    print "The following files should be scheduled for deletion with p4 delete:"
 601                    print " ".join(filesToDelete)
 602                die("Please resolve and submit the conflict manually and "
 603                    + "continue afterwards with git-p4 submit --continue")
 604            elif response == "w":
 605                system(diffcmd + " > patch.txt")
 606                print "Patch saved to patch.txt in %s !" % self.clientPath
 607                die("Please resolve and submit the conflict manually and "
 608                    "continue afterwards with git-p4 submit --continue")
 609
 610        system(applyPatchCmd)
 611
 612        for f in filesToAdd:
 613            system("p4 add \"%s\"" % f)
 614        for f in filesToDelete:
 615            system("p4 revert \"%s\"" % f)
 616            system("p4 delete \"%s\"" % f)
 617
 618        # Set/clear executable bits
 619        for f in filesToChangeExecBit.keys():
 620            mode = filesToChangeExecBit[f]
 621            setP4ExecBit(f, mode)
 622
 623        logMessage = extractLogMessageFromGitCommit(id)
 624        logMessage = logMessage.strip()
 625
 626        template = self.prepareSubmitTemplate()
 627
 628        if self.interactive:
 629            submitTemplate = self.prepareLogMessage(template, logMessage)
 630            if os.environ.has_key("P4DIFF"):
 631                del(os.environ["P4DIFF"])
 632            diff = read_pipe("p4 diff -du ...")
 633
 634            newdiff = ""
 635            for newFile in filesToAdd:
 636                newdiff += "==== new file ====\n"
 637                newdiff += "--- /dev/null\n"
 638                newdiff += "+++ %s\n" % newFile
 639                f = open(newFile, "r")
 640                for line in f.readlines():
 641                    newdiff += "+" + line
 642                f.close()
 643
 644            separatorLine = "######## everything below this line is just the diff #######\n"
 645
 646            [handle, fileName] = tempfile.mkstemp()
 647            tmpFile = os.fdopen(handle, "w+")
 648            if self.isWindows:
 649                submitTemplate = submitTemplate.replace("\n", "\r\n")
 650                separatorLine = separatorLine.replace("\n", "\r\n")
 651                newdiff = newdiff.replace("\n", "\r\n")
 652            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 653            tmpFile.close()
 654            defaultEditor = "vi"
 655            if platform.system() == "Windows":
 656                defaultEditor = "notepad"
 657            if os.environ.has_key("P4EDITOR"):
 658                editor = os.environ.get("P4EDITOR")
 659            else:
 660                editor = os.environ.get("EDITOR", defaultEditor);
 661            system(editor + " " + fileName)
 662            tmpFile = open(fileName, "rb")
 663            message = tmpFile.read()
 664            tmpFile.close()
 665            os.remove(fileName)
 666            submitTemplate = message[:message.index(separatorLine)]
 667            if self.isWindows:
 668                submitTemplate = submitTemplate.replace("\r\n", "\n")
 669
 670            write_pipe("p4 submit -i", submitTemplate)
 671        else:
 672            fileName = "submit.txt"
 673            file = open(fileName, "w+")
 674            file.write(self.prepareLogMessage(template, logMessage))
 675            file.close()
 676            print ("Perforce submit template written as %s. "
 677                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 678                   % (fileName, fileName))
 679
 680    def run(self, args):
 681        if len(args) == 0:
 682            self.master = currentGitBranch()
 683            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 684                die("Detecting current git branch failed!")
 685        elif len(args) == 1:
 686            self.master = args[0]
 687        else:
 688            return False
 689
 690        allowSubmit = gitConfig("git-p4.allowSubmit")
 691        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 692            die("%s is not in git-p4.allowSubmit" % self.master)
 693
 694        [upstream, settings] = findUpstreamBranchPoint()
 695        self.depotPath = settings['depot-paths'][0]
 696        if len(self.origin) == 0:
 697            self.origin = upstream
 698
 699        if self.verbose:
 700            print "Origin branch is " + self.origin
 701
 702        if len(self.depotPath) == 0:
 703            print "Internal error: cannot locate perforce depot path from existing branches"
 704            sys.exit(128)
 705
 706        self.clientPath = p4Where(self.depotPath)
 707
 708        if len(self.clientPath) == 0:
 709            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 710            sys.exit(128)
 711
 712        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 713        self.oldWorkingDirectory = os.getcwd()
 714
 715        os.chdir(self.clientPath)
 716        print "Syncronizing p4 checkout..."
 717        system("p4 sync ...")
 718
 719        self.check()
 720
 721        commits = []
 722        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 723            commits.append(line.strip())
 724        commits.reverse()
 725
 726        while len(commits) > 0:
 727            commit = commits[0]
 728            commits = commits[1:]
 729            self.applyCommit(commit)
 730            if not self.interactive:
 731                break
 732
 733        if len(commits) == 0:
 734            print "All changes applied!"
 735            os.chdir(self.oldWorkingDirectory)
 736
 737            sync = P4Sync()
 738            sync.run([])
 739
 740            rebase = P4Rebase()
 741            rebase.rebase()
 742
 743        return True
 744
 745class P4Sync(Command):
 746    def __init__(self):
 747        Command.__init__(self)
 748        self.options = [
 749                optparse.make_option("--branch", dest="branch"),
 750                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 751                optparse.make_option("--changesfile", dest="changesFile"),
 752                optparse.make_option("--silent", dest="silent", action="store_true"),
 753                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 754                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 755                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 756                                     help="Import into refs/heads/ , not refs/remotes"),
 757                optparse.make_option("--max-changes", dest="maxChanges"),
 758                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 759                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 760                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 761                                     help="Only sync files that are included in the Perforce Client Spec")
 762        ]
 763        self.description = """Imports from Perforce into a git repository.\n
 764    example:
 765    //depot/my/project/ -- to import the current head
 766    //depot/my/project/@all -- to import everything
 767    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 768
 769    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 770
 771        self.usage += " //depot/path[@revRange]"
 772        self.silent = False
 773        self.createdBranches = Set()
 774        self.committedChanges = Set()
 775        self.branch = ""
 776        self.detectBranches = False
 777        self.detectLabels = False
 778        self.changesFile = ""
 779        self.syncWithOrigin = True
 780        self.verbose = False
 781        self.importIntoRemotes = True
 782        self.maxChanges = ""
 783        self.isWindows = (platform.system() == "Windows")
 784        self.keepRepoPath = False
 785        self.depotPaths = None
 786        self.p4BranchesInGit = []
 787        self.cloneExclude = []
 788        self.useClientSpec = False
 789        self.clientSpecDirs = []
 790
 791        if gitConfig("git-p4.syncFromOrigin") == "false":
 792            self.syncWithOrigin = False
 793
 794    def extractFilesFromCommit(self, commit):
 795        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 796                             for path in self.cloneExclude]
 797        files = []
 798        fnum = 0
 799        while commit.has_key("depotFile%s" % fnum):
 800            path =  commit["depotFile%s" % fnum]
 801
 802            if [p for p in self.cloneExclude
 803                if path.startswith (p)]:
 804                found = False
 805            else:
 806                found = [p for p in self.depotPaths
 807                         if path.startswith (p)]
 808            if not found:
 809                fnum = fnum + 1
 810                continue
 811
 812            file = {}
 813            file["path"] = path
 814            file["rev"] = commit["rev%s" % fnum]
 815            file["action"] = commit["action%s" % fnum]
 816            file["type"] = commit["type%s" % fnum]
 817            files.append(file)
 818            fnum = fnum + 1
 819        return files
 820
 821    def stripRepoPath(self, path, prefixes):
 822        if self.keepRepoPath:
 823            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 824
 825        for p in prefixes:
 826            if path.startswith(p):
 827                path = path[len(p):]
 828
 829        return path
 830
 831    def splitFilesIntoBranches(self, commit):
 832        branches = {}
 833        fnum = 0
 834        while commit.has_key("depotFile%s" % fnum):
 835            path =  commit["depotFile%s" % fnum]
 836            found = [p for p in self.depotPaths
 837                     if path.startswith (p)]
 838            if not found:
 839                fnum = fnum + 1
 840                continue
 841
 842            file = {}
 843            file["path"] = path
 844            file["rev"] = commit["rev%s" % fnum]
 845            file["action"] = commit["action%s" % fnum]
 846            file["type"] = commit["type%s" % fnum]
 847            fnum = fnum + 1
 848
 849            relPath = self.stripRepoPath(path, self.depotPaths)
 850
 851            for branch in self.knownBranches.keys():
 852
 853                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 854                if relPath.startswith(branch + "/"):
 855                    if branch not in branches:
 856                        branches[branch] = []
 857                    branches[branch].append(file)
 858                    break
 859
 860        return branches
 861
 862    ## Should move this out, doesn't use SELF.
 863    def readP4Files(self, files):
 864        filesForCommit = []
 865        filesToRead = []
 866
 867        for f in files:
 868            includeFile = True
 869            for val in self.clientSpecDirs:
 870                if f['path'].startswith(val[0]):
 871                    if val[1] <= 0:
 872                        includeFile = False
 873                    break
 874
 875            if includeFile:
 876                filesForCommit.append(f)
 877                if f['action'] != 'delete':
 878                    filesToRead.append(f)
 879
 880        filedata = []
 881        if len(filesToRead) > 0:
 882            filedata = p4CmdList('-x - print',
 883                                 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 884                                                  for f in filesToRead]),
 885                                 stdin_mode='w+')
 886
 887            if "p4ExitCode" in filedata[0]:
 888                die("Problems executing p4. Error: [%d]."
 889                    % (filedata[0]['p4ExitCode']));
 890
 891        j = 0;
 892        contents = {}
 893        while j < len(filedata):
 894            stat = filedata[j]
 895            j += 1
 896            text = [];
 897            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 898                text.append(filedata[j]['data'])
 899                j += 1
 900            text = ''.join(text)
 901
 902            if not stat.has_key('depotFile'):
 903                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 904                continue
 905
 906            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 907                text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 908            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 909                text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 910
 911            contents[stat['depotFile']] = text
 912
 913        for f in filesForCommit:
 914            path = f['path']
 915            if contents.has_key(path):
 916                f['data'] = contents[path]
 917
 918        return filesForCommit
 919
 920    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 921        epoch = details["time"]
 922        author = details["user"]
 923
 924        if self.verbose:
 925            print "commit into %s" % branch
 926
 927        # start with reading files; if that fails, we should not
 928        # create a commit.
 929        new_files = []
 930        for f in files:
 931            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 932                new_files.append (f)
 933            else:
 934                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 935        files = self.readP4Files(new_files)
 936
 937        self.gitStream.write("commit %s\n" % branch)
 938#        gitStream.write("mark :%s\n" % details["change"])
 939        self.committedChanges.add(int(details["change"]))
 940        committer = ""
 941        if author not in self.users:
 942            self.getUserMapFromPerforceServer()
 943        if author in self.users:
 944            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 945        else:
 946            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 947
 948        self.gitStream.write("committer %s\n" % committer)
 949
 950        self.gitStream.write("data <<EOT\n")
 951        self.gitStream.write(details["desc"])
 952        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 953                             % (','.join (branchPrefixes), details["change"]))
 954        if len(details['options']) > 0:
 955            self.gitStream.write(": options = %s" % details['options'])
 956        self.gitStream.write("]\nEOT\n\n")
 957
 958        if len(parent) > 0:
 959            if self.verbose:
 960                print "parent %s" % parent
 961            self.gitStream.write("from %s\n" % parent)
 962
 963        for file in files:
 964            if file["type"] == "apple":
 965                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 966                continue
 967
 968            relPath = self.stripRepoPath(file['path'], branchPrefixes)
 969            if file["action"] == "delete":
 970                self.gitStream.write("D %s\n" % relPath)
 971            else:
 972                data = file['data']
 973
 974                mode = "644"
 975                if isP4Exec(file["type"]):
 976                    mode = "755"
 977                elif file["type"] == "symlink":
 978                    mode = "120000"
 979                    # p4 print on a symlink contains "target\n", so strip it off
 980                    data = data[:-1]
 981
 982                if self.isWindows and file["type"].endswith("text"):
 983                    data = data.replace("\r\n", "\n")
 984
 985                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 986                self.gitStream.write("data %s\n" % len(data))
 987                self.gitStream.write(data)
 988                self.gitStream.write("\n")
 989
 990        self.gitStream.write("\n")
 991
 992        change = int(details["change"])
 993
 994        if self.labels.has_key(change):
 995            label = self.labels[change]
 996            labelDetails = label[0]
 997            labelRevisions = label[1]
 998            if self.verbose:
 999                print "Change %s is labelled %s" % (change, labelDetails)
1000
1001            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1002                                                    for p in branchPrefixes]))
1003
1004            if len(files) == len(labelRevisions):
1005
1006                cleanedFiles = {}
1007                for info in files:
1008                    if info["action"] == "delete":
1009                        continue
1010                    cleanedFiles[info["depotFile"]] = info["rev"]
1011
1012                if cleanedFiles == labelRevisions:
1013                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1014                    self.gitStream.write("from %s\n" % branch)
1015
1016                    owner = labelDetails["Owner"]
1017                    tagger = ""
1018                    if author in self.users:
1019                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1020                    else:
1021                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1022                    self.gitStream.write("tagger %s\n" % tagger)
1023                    self.gitStream.write("data <<EOT\n")
1024                    self.gitStream.write(labelDetails["Description"])
1025                    self.gitStream.write("EOT\n\n")
1026
1027                else:
1028                    if not self.silent:
1029                        print ("Tag %s does not match with change %s: files do not match."
1030                               % (labelDetails["label"], change))
1031
1032            else:
1033                if not self.silent:
1034                    print ("Tag %s does not match with change %s: file count is different."
1035                           % (labelDetails["label"], change))
1036
1037    def getUserCacheFilename(self):
1038        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1039        return home + "/.gitp4-usercache.txt"
1040
1041    def getUserMapFromPerforceServer(self):
1042        if self.userMapFromPerforceServer:
1043            return
1044        self.users = {}
1045
1046        for output in p4CmdList("users"):
1047            if not output.has_key("User"):
1048                continue
1049            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1050
1051
1052        s = ''
1053        for (key, val) in self.users.items():
1054            s += "%s\t%s\n" % (key, val)
1055
1056        open(self.getUserCacheFilename(), "wb").write(s)
1057        self.userMapFromPerforceServer = True
1058
1059    def loadUserMapFromCache(self):
1060        self.users = {}
1061        self.userMapFromPerforceServer = False
1062        try:
1063            cache = open(self.getUserCacheFilename(), "rb")
1064            lines = cache.readlines()
1065            cache.close()
1066            for line in lines:
1067                entry = line.strip().split("\t")
1068                self.users[entry[0]] = entry[1]
1069        except IOError:
1070            self.getUserMapFromPerforceServer()
1071
1072    def getLabels(self):
1073        self.labels = {}
1074
1075        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1076        if len(l) > 0 and not self.silent:
1077            print "Finding files belonging to labels in %s" % `self.depotPaths`
1078
1079        for output in l:
1080            label = output["label"]
1081            revisions = {}
1082            newestChange = 0
1083            if self.verbose:
1084                print "Querying files for label %s" % label
1085            for file in p4CmdList("files "
1086                                  +  ' '.join (["%s...@%s" % (p, label)
1087                                                for p in self.depotPaths])):
1088                revisions[file["depotFile"]] = file["rev"]
1089                change = int(file["change"])
1090                if change > newestChange:
1091                    newestChange = change
1092
1093            self.labels[newestChange] = [output, revisions]
1094
1095        if self.verbose:
1096            print "Label changes: %s" % self.labels.keys()
1097
1098    def guessProjectName(self):
1099        for p in self.depotPaths:
1100            if p.endswith("/"):
1101                p = p[:-1]
1102            p = p[p.strip().rfind("/") + 1:]
1103            if not p.endswith("/"):
1104               p += "/"
1105            return p
1106
1107    def getBranchMapping(self):
1108        lostAndFoundBranches = set()
1109
1110        for info in p4CmdList("branches"):
1111            details = p4Cmd("branch -o %s" % info["branch"])
1112            viewIdx = 0
1113            while details.has_key("View%s" % viewIdx):
1114                paths = details["View%s" % viewIdx].split(" ")
1115                viewIdx = viewIdx + 1
1116                # require standard //depot/foo/... //depot/bar/... mapping
1117                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1118                    continue
1119                source = paths[0]
1120                destination = paths[1]
1121                ## HACK
1122                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1123                    source = source[len(self.depotPaths[0]):-4]
1124                    destination = destination[len(self.depotPaths[0]):-4]
1125
1126                    if destination in self.knownBranches:
1127                        if not self.silent:
1128                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1129                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1130                        continue
1131
1132                    self.knownBranches[destination] = source
1133
1134                    lostAndFoundBranches.discard(destination)
1135
1136                    if source not in self.knownBranches:
1137                        lostAndFoundBranches.add(source)
1138
1139
1140        for branch in lostAndFoundBranches:
1141            self.knownBranches[branch] = branch
1142
1143    def getBranchMappingFromGitBranches(self):
1144        branches = p4BranchesInGit(self.importIntoRemotes)
1145        for branch in branches.keys():
1146            if branch == "master":
1147                branch = "main"
1148            else:
1149                branch = branch[len(self.projectName):]
1150            self.knownBranches[branch] = branch
1151
1152    def listExistingP4GitBranches(self):
1153        # branches holds mapping from name to commit
1154        branches = p4BranchesInGit(self.importIntoRemotes)
1155        self.p4BranchesInGit = branches.keys()
1156        for branch in branches.keys():
1157            self.initialParents[self.refPrefix + branch] = branches[branch]
1158
1159    def updateOptionDict(self, d):
1160        option_keys = {}
1161        if self.keepRepoPath:
1162            option_keys['keepRepoPath'] = 1
1163
1164        d["options"] = ' '.join(sorted(option_keys.keys()))
1165
1166    def readOptions(self, d):
1167        self.keepRepoPath = (d.has_key('options')
1168                             and ('keepRepoPath' in d['options']))
1169
1170    def gitRefForBranch(self, branch):
1171        if branch == "main":
1172            return self.refPrefix + "master"
1173
1174        if len(branch) <= 0:
1175            return branch
1176
1177        return self.refPrefix + self.projectName + branch
1178
1179    def gitCommitByP4Change(self, ref, change):
1180        if self.verbose:
1181            print "looking in ref " + ref + " for change %s using bisect..." % change
1182
1183        earliestCommit = ""
1184        latestCommit = parseRevision(ref)
1185
1186        while True:
1187            if self.verbose:
1188                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1189            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1190            if len(next) == 0:
1191                if self.verbose:
1192                    print "argh"
1193                return ""
1194            log = extractLogMessageFromGitCommit(next)
1195            settings = extractSettingsGitLog(log)
1196            currentChange = int(settings['change'])
1197            if self.verbose:
1198                print "current change %s" % currentChange
1199
1200            if currentChange == change:
1201                if self.verbose:
1202                    print "found %s" % next
1203                return next
1204
1205            if currentChange < change:
1206                earliestCommit = "^%s" % next
1207            else:
1208                latestCommit = "%s" % next
1209
1210        return ""
1211
1212    def importNewBranch(self, branch, maxChange):
1213        # make fast-import flush all changes to disk and update the refs using the checkpoint
1214        # command so that we can try to find the branch parent in the git history
1215        self.gitStream.write("checkpoint\n\n");
1216        self.gitStream.flush();
1217        branchPrefix = self.depotPaths[0] + branch + "/"
1218        range = "@1,%s" % maxChange
1219        #print "prefix" + branchPrefix
1220        changes = p4ChangesForPaths([branchPrefix], range)
1221        if len(changes) <= 0:
1222            return False
1223        firstChange = changes[0]
1224        #print "first change in branch: %s" % firstChange
1225        sourceBranch = self.knownBranches[branch]
1226        sourceDepotPath = self.depotPaths[0] + sourceBranch
1227        sourceRef = self.gitRefForBranch(sourceBranch)
1228        #print "source " + sourceBranch
1229
1230        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1231        #print "branch parent: %s" % branchParentChange
1232        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1233        if len(gitParent) > 0:
1234            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1235            #print "parent git commit: %s" % gitParent
1236
1237        self.importChanges(changes)
1238        return True
1239
1240    def importChanges(self, changes):
1241        cnt = 1
1242        for change in changes:
1243            description = p4Cmd("describe %s" % change)
1244            self.updateOptionDict(description)
1245
1246            if not self.silent:
1247                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1248                sys.stdout.flush()
1249            cnt = cnt + 1
1250
1251            try:
1252                if self.detectBranches:
1253                    branches = self.splitFilesIntoBranches(description)
1254                    for branch in branches.keys():
1255                        ## HACK  --hwn
1256                        branchPrefix = self.depotPaths[0] + branch + "/"
1257
1258                        parent = ""
1259
1260                        filesForCommit = branches[branch]
1261
1262                        if self.verbose:
1263                            print "branch is %s" % branch
1264
1265                        self.updatedBranches.add(branch)
1266
1267                        if branch not in self.createdBranches:
1268                            self.createdBranches.add(branch)
1269                            parent = self.knownBranches[branch]
1270                            if parent == branch:
1271                                parent = ""
1272                            else:
1273                                fullBranch = self.projectName + branch
1274                                if fullBranch not in self.p4BranchesInGit:
1275                                    if not self.silent:
1276                                        print("\n    Importing new branch %s" % fullBranch);
1277                                    if self.importNewBranch(branch, change - 1):
1278                                        parent = ""
1279                                        self.p4BranchesInGit.append(fullBranch)
1280                                    if not self.silent:
1281                                        print("\n    Resuming with change %s" % change);
1282
1283                                if self.verbose:
1284                                    print "parent determined through known branches: %s" % parent
1285
1286                        branch = self.gitRefForBranch(branch)
1287                        parent = self.gitRefForBranch(parent)
1288
1289                        if self.verbose:
1290                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1291
1292                        if len(parent) == 0 and branch in self.initialParents:
1293                            parent = self.initialParents[branch]
1294                            del self.initialParents[branch]
1295
1296                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1297                else:
1298                    files = self.extractFilesFromCommit(description)
1299                    self.commit(description, files, self.branch, self.depotPaths,
1300                                self.initialParent)
1301                    self.initialParent = ""
1302            except IOError:
1303                print self.gitError.read()
1304                sys.exit(1)
1305
1306    def importHeadRevision(self, revision):
1307        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1308
1309        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1310        details["desc"] = ("Initial import of %s from the state at revision %s"
1311                           % (' '.join(self.depotPaths), revision))
1312        details["change"] = revision
1313        newestRevision = 0
1314
1315        fileCnt = 0
1316        for info in p4CmdList("files "
1317                              +  ' '.join(["%s...%s"
1318                                           % (p, revision)
1319                                           for p in self.depotPaths])):
1320
1321            if info['code'] == 'error':
1322                sys.stderr.write("p4 returned an error: %s\n"
1323                                 % info['data'])
1324                sys.exit(1)
1325
1326
1327            change = int(info["change"])
1328            if change > newestRevision:
1329                newestRevision = change
1330
1331            if info["action"] == "delete":
1332                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1333                #fileCnt = fileCnt + 1
1334                continue
1335
1336            for prop in ["depotFile", "rev", "action", "type" ]:
1337                details["%s%s" % (prop, fileCnt)] = info[prop]
1338
1339            fileCnt = fileCnt + 1
1340
1341        details["change"] = newestRevision
1342        self.updateOptionDict(details)
1343        try:
1344            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1345        except IOError:
1346            print "IO error with git fast-import. Is your git version recent enough?"
1347            print self.gitError.read()
1348
1349
1350    def getClientSpec(self):
1351        specList = p4CmdList( "client -o" )
1352        temp = {}
1353        for entry in specList:
1354            for k,v in entry.iteritems():
1355                if k.startswith("View"):
1356                    if v.startswith('"'):
1357                        start = 1
1358                    else:
1359                        start = 0
1360                    index = v.find("...")
1361                    v = v[start:index]
1362                    if v.startswith("-"):
1363                        v = v[1:]
1364                        temp[v] = -len(v)
1365                    else:
1366                        temp[v] = len(v)
1367        self.clientSpecDirs = temp.items()
1368        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1369
1370    def run(self, args):
1371        self.depotPaths = []
1372        self.changeRange = ""
1373        self.initialParent = ""
1374        self.previousDepotPaths = []
1375
1376        # map from branch depot path to parent branch
1377        self.knownBranches = {}
1378        self.initialParents = {}
1379        self.hasOrigin = originP4BranchesExist()
1380        if not self.syncWithOrigin:
1381            self.hasOrigin = False
1382
1383        if self.importIntoRemotes:
1384            self.refPrefix = "refs/remotes/p4/"
1385        else:
1386            self.refPrefix = "refs/heads/p4/"
1387
1388        if self.syncWithOrigin and self.hasOrigin:
1389            if not self.silent:
1390                print "Syncing with origin first by calling git fetch origin"
1391            system("git fetch origin")
1392
1393        if len(self.branch) == 0:
1394            self.branch = self.refPrefix + "master"
1395            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1396                system("git update-ref %s refs/heads/p4" % self.branch)
1397                system("git branch -D p4");
1398            # create it /after/ importing, when master exists
1399            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1400                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1401
1402        if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1403            self.getClientSpec()
1404
1405        # TODO: should always look at previous commits,
1406        # merge with previous imports, if possible.
1407        if args == []:
1408            if self.hasOrigin:
1409                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1410            self.listExistingP4GitBranches()
1411
1412            if len(self.p4BranchesInGit) > 1:
1413                if not self.silent:
1414                    print "Importing from/into multiple branches"
1415                self.detectBranches = True
1416
1417            if self.verbose:
1418                print "branches: %s" % self.p4BranchesInGit
1419
1420            p4Change = 0
1421            for branch in self.p4BranchesInGit:
1422                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1423
1424                settings = extractSettingsGitLog(logMsg)
1425
1426                self.readOptions(settings)
1427                if (settings.has_key('depot-paths')
1428                    and settings.has_key ('change')):
1429                    change = int(settings['change']) + 1
1430                    p4Change = max(p4Change, change)
1431
1432                    depotPaths = sorted(settings['depot-paths'])
1433                    if self.previousDepotPaths == []:
1434                        self.previousDepotPaths = depotPaths
1435                    else:
1436                        paths = []
1437                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1438                            for i in range(0, min(len(cur), len(prev))):
1439                                if cur[i] <> prev[i]:
1440                                    i = i - 1
1441                                    break
1442
1443                            paths.append (cur[:i + 1])
1444
1445                        self.previousDepotPaths = paths
1446
1447            if p4Change > 0:
1448                self.depotPaths = sorted(self.previousDepotPaths)
1449                self.changeRange = "@%s,#head" % p4Change
1450                if not self.detectBranches:
1451                    self.initialParent = parseRevision(self.branch)
1452                if not self.silent and not self.detectBranches:
1453                    print "Performing incremental import into %s git branch" % self.branch
1454
1455        if not self.branch.startswith("refs/"):
1456            self.branch = "refs/heads/" + self.branch
1457
1458        if len(args) == 0 and self.depotPaths:
1459            if not self.silent:
1460                print "Depot paths: %s" % ' '.join(self.depotPaths)
1461        else:
1462            if self.depotPaths and self.depotPaths != args:
1463                print ("previous import used depot path %s and now %s was specified. "
1464                       "This doesn't work!" % (' '.join (self.depotPaths),
1465                                               ' '.join (args)))
1466                sys.exit(1)
1467
1468            self.depotPaths = sorted(args)
1469
1470        revision = ""
1471        self.users = {}
1472
1473        newPaths = []
1474        for p in self.depotPaths:
1475            if p.find("@") != -1:
1476                atIdx = p.index("@")
1477                self.changeRange = p[atIdx:]
1478                if self.changeRange == "@all":
1479                    self.changeRange = ""
1480                elif ',' not in self.changeRange:
1481                    revision = self.changeRange
1482                    self.changeRange = ""
1483                p = p[:atIdx]
1484            elif p.find("#") != -1:
1485                hashIdx = p.index("#")
1486                revision = p[hashIdx:]
1487                p = p[:hashIdx]
1488            elif self.previousDepotPaths == []:
1489                revision = "#head"
1490
1491            p = re.sub ("\.\.\.$", "", p)
1492            if not p.endswith("/"):
1493                p += "/"
1494
1495            newPaths.append(p)
1496
1497        self.depotPaths = newPaths
1498
1499
1500        self.loadUserMapFromCache()
1501        self.labels = {}
1502        if self.detectLabels:
1503            self.getLabels();
1504
1505        if self.detectBranches:
1506            ## FIXME - what's a P4 projectName ?
1507            self.projectName = self.guessProjectName()
1508
1509            if self.hasOrigin:
1510                self.getBranchMappingFromGitBranches()
1511            else:
1512                self.getBranchMapping()
1513            if self.verbose:
1514                print "p4-git branches: %s" % self.p4BranchesInGit
1515                print "initial parents: %s" % self.initialParents
1516            for b in self.p4BranchesInGit:
1517                if b != "master":
1518
1519                    ## FIXME
1520                    b = b[len(self.projectName):]
1521                self.createdBranches.add(b)
1522
1523        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1524
1525        importProcess = subprocess.Popen(["git", "fast-import"],
1526                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1527                                         stderr=subprocess.PIPE);
1528        self.gitOutput = importProcess.stdout
1529        self.gitStream = importProcess.stdin
1530        self.gitError = importProcess.stderr
1531
1532        if revision:
1533            self.importHeadRevision(revision)
1534        else:
1535            changes = []
1536
1537            if len(self.changesFile) > 0:
1538                output = open(self.changesFile).readlines()
1539                changeSet = Set()
1540                for line in output:
1541                    changeSet.add(int(line))
1542
1543                for change in changeSet:
1544                    changes.append(change)
1545
1546                changes.sort()
1547            else:
1548                if self.verbose:
1549                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1550                                                              self.changeRange)
1551                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1552
1553                if len(self.maxChanges) > 0:
1554                    changes = changes[:min(int(self.maxChanges), len(changes))]
1555
1556            if len(changes) == 0:
1557                if not self.silent:
1558                    print "No changes to import!"
1559                return True
1560
1561            if not self.silent and not self.detectBranches:
1562                print "Import destination: %s" % self.branch
1563
1564            self.updatedBranches = set()
1565
1566            self.importChanges(changes)
1567
1568            if not self.silent:
1569                print ""
1570                if len(self.updatedBranches) > 0:
1571                    sys.stdout.write("Updated branches: ")
1572                    for b in self.updatedBranches:
1573                        sys.stdout.write("%s " % b)
1574                    sys.stdout.write("\n")
1575
1576        self.gitStream.close()
1577        if importProcess.wait() != 0:
1578            die("fast-import failed: %s" % self.gitError.read())
1579        self.gitOutput.close()
1580        self.gitError.close()
1581
1582        return True
1583
1584class P4Rebase(Command):
1585    def __init__(self):
1586        Command.__init__(self)
1587        self.options = [ ]
1588        self.description = ("Fetches the latest revision from perforce and "
1589                            + "rebases the current work (branch) against it")
1590        self.verbose = False
1591
1592    def run(self, args):
1593        sync = P4Sync()
1594        sync.run([])
1595
1596        return self.rebase()
1597
1598    def rebase(self):
1599        if os.system("git update-index --refresh") != 0:
1600            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1601        if len(read_pipe("git diff-index HEAD --")) > 0:
1602            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1603
1604        [upstream, settings] = findUpstreamBranchPoint()
1605        if len(upstream) == 0:
1606            die("Cannot find upstream branchpoint for rebase")
1607
1608        # the branchpoint may be p4/foo~3, so strip off the parent
1609        upstream = re.sub("~[0-9]+$", "", upstream)
1610
1611        print "Rebasing the current branch onto %s" % upstream
1612        oldHead = read_pipe("git rev-parse HEAD").strip()
1613        system("git rebase %s" % upstream)
1614        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1615        return True
1616
1617class P4Clone(P4Sync):
1618    def __init__(self):
1619        P4Sync.__init__(self)
1620        self.description = "Creates a new git repository and imports from Perforce into it"
1621        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1622        self.options += [
1623            optparse.make_option("--destination", dest="cloneDestination",
1624                                 action='store', default=None,
1625                                 help="where to leave result of the clone"),
1626            optparse.make_option("-/", dest="cloneExclude",
1627                                 action="append", type="string",
1628                                 help="exclude depot path")
1629        ]
1630        self.cloneDestination = None
1631        self.needsGit = False
1632
1633    # This is required for the "append" cloneExclude action
1634    def ensure_value(self, attr, value):
1635        if not hasattr(self, attr) or getattr(self, attr) is None:
1636            setattr(self, attr, value)
1637        return getattr(self, attr)
1638
1639    def defaultDestination(self, args):
1640        ## TODO: use common prefix of args?
1641        depotPath = args[0]
1642        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1643        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1644        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1645        depotDir = re.sub(r"/$", "", depotDir)
1646        return os.path.split(depotDir)[1]
1647
1648    def run(self, args):
1649        if len(args) < 1:
1650            return False
1651
1652        if self.keepRepoPath and not self.cloneDestination:
1653            sys.stderr.write("Must specify destination for --keep-path\n")
1654            sys.exit(1)
1655
1656        depotPaths = args
1657
1658        if not self.cloneDestination and len(depotPaths) > 1:
1659            self.cloneDestination = depotPaths[-1]
1660            depotPaths = depotPaths[:-1]
1661
1662        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1663        for p in depotPaths:
1664            if not p.startswith("//"):
1665                return False
1666
1667        if not self.cloneDestination:
1668            self.cloneDestination = self.defaultDestination(args)
1669
1670        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1671        if not os.path.exists(self.cloneDestination):
1672            os.makedirs(self.cloneDestination)
1673        os.chdir(self.cloneDestination)
1674        system("git init")
1675        self.gitdir = os.getcwd() + "/.git"
1676        if not P4Sync.run(self, depotPaths):
1677            return False
1678        if self.branch != "master":
1679            if gitBranchExists("refs/remotes/p4/master"):
1680                system("git branch master refs/remotes/p4/master")
1681                system("git checkout -f")
1682            else:
1683                print "Could not detect main branch. No checkout/master branch created."
1684
1685        return True
1686
1687class P4Branches(Command):
1688    def __init__(self):
1689        Command.__init__(self)
1690        self.options = [ ]
1691        self.description = ("Shows the git branches that hold imports and their "
1692                            + "corresponding perforce depot paths")
1693        self.verbose = False
1694
1695    def run(self, args):
1696        if originP4BranchesExist():
1697            createOrUpdateBranchesFromOrigin()
1698
1699        cmdline = "git rev-parse --symbolic "
1700        cmdline += " --remotes"
1701
1702        for line in read_pipe_lines(cmdline):
1703            line = line.strip()
1704
1705            if not line.startswith('p4/') or line == "p4/HEAD":
1706                continue
1707            branch = line
1708
1709            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1710            settings = extractSettingsGitLog(log)
1711
1712            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1713        return True
1714
1715class HelpFormatter(optparse.IndentedHelpFormatter):
1716    def __init__(self):
1717        optparse.IndentedHelpFormatter.__init__(self)
1718
1719    def format_description(self, description):
1720        if description:
1721            return description + "\n"
1722        else:
1723            return ""
1724
1725def printUsage(commands):
1726    print "usage: %s <command> [options]" % sys.argv[0]
1727    print ""
1728    print "valid commands: %s" % ", ".join(commands)
1729    print ""
1730    print "Try %s <command> --help for command specific help." % sys.argv[0]
1731    print ""
1732
1733commands = {
1734    "debug" : P4Debug,
1735    "submit" : P4Submit,
1736    "commit" : P4Submit,
1737    "sync" : P4Sync,
1738    "rebase" : P4Rebase,
1739    "clone" : P4Clone,
1740    "rollback" : P4RollBack,
1741    "branches" : P4Branches
1742}
1743
1744
1745def main():
1746    if len(sys.argv[1:]) == 0:
1747        printUsage(commands.keys())
1748        sys.exit(2)
1749
1750    cmd = ""
1751    cmdName = sys.argv[1]
1752    try:
1753        klass = commands[cmdName]
1754        cmd = klass()
1755    except KeyError:
1756        print "unknown command %s" % cmdName
1757        print ""
1758        printUsage(commands.keys())
1759        sys.exit(2)
1760
1761    options = cmd.options
1762    cmd.gitdir = os.environ.get("GIT_DIR", None)
1763
1764    args = sys.argv[2:]
1765
1766    if len(options) > 0:
1767        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1768
1769        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1770                                       options,
1771                                       description = cmd.description,
1772                                       formatter = HelpFormatter())
1773
1774        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1775    global verbose
1776    verbose = cmd.verbose
1777    if cmd.needsGit:
1778        if cmd.gitdir == None:
1779            cmd.gitdir = os.path.abspath(".git")
1780            if not isValidGitDir(cmd.gitdir):
1781                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1782                if os.path.exists(cmd.gitdir):
1783                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1784                    if len(cdup) > 0:
1785                        os.chdir(cdup);
1786
1787        if not isValidGitDir(cmd.gitdir):
1788            if isValidGitDir(cmd.gitdir + "/.git"):
1789                cmd.gitdir += "/.git"
1790            else:
1791                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1792
1793        os.environ["GIT_DIR"] = cmd.gitdir
1794
1795    if not cmd.run(args):
1796        parser.print_help()
1797
1798
1799if __name__ == '__main__':
1800    main()