Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'db/remote-builtin' into jk/send-pack (a108e53)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19def die(msg):
  20    if verbose:
  21        raise Exception(msg)
  22    else:
  23        sys.stderr.write(msg + "\n")
  24        sys.exit(1)
  25
  26def write_pipe(c, str):
  27    if verbose:
  28        sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30    pipe = os.popen(c, 'w')
  31    val = pipe.write(str)
  32    if pipe.close():
  33        die('Command failed: %s' % c)
  34
  35    return val
  36
  37def read_pipe(c, ignore_error=False):
  38    if verbose:
  39        sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41    pipe = os.popen(c, 'rb')
  42    val = pipe.read()
  43    if pipe.close() and not ignore_error:
  44        die('Command failed: %s' % c)
  45
  46    return val
  47
  48
  49def read_pipe_lines(c):
  50    if verbose:
  51        sys.stderr.write('Reading pipe: %s\n' % c)
  52    ## todo: check return status
  53    pipe = os.popen(c, 'rb')
  54    val = pipe.readlines()
  55    if pipe.close():
  56        die('Command failed: %s' % c)
  57
  58    return val
  59
  60def system(cmd):
  61    if verbose:
  62        sys.stderr.write("executing %s\n" % cmd)
  63    if os.system(cmd) != 0:
  64        die("command failed: %s" % cmd)
  65
  66def isP4Exec(kind):
  67    """Determine if a Perforce 'kind' should have execute permission
  68
  69    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70    or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71    a plus sign, it is also executable"""
  72    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74def setP4ExecBit(file, mode):
  75    # Reopens an already open file and changes the execute bit to match
  76    # the execute bit setting in the passed in mode.
  77
  78    p4Type = "+x"
  79
  80    if not isModeExec(mode):
  81        p4Type = getP4OpenedType(file)
  82        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84        if p4Type[-1] == "+":
  85            p4Type = p4Type[0:-1]
  86
  87    system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89def getP4OpenedType(file):
  90    # Returns the perforce file type for the given file.
  91
  92    result = read_pipe("p4 opened %s" % file)
  93    match = re.match(".*\((.+)\)$", result)
  94    if match:
  95        return match.group(1)
  96    else:
  97        die("Could not determine file type for %s" % file)
  98
  99def diffTreePattern():
 100    # This is a simple generator for the diff tree regex pattern. This could be
 101    # a class variable if this and parseDiffTreeEntry were a part of a class.
 102    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103    while True:
 104        yield pattern
 105
 106def parseDiffTreeEntry(entry):
 107    """Parses a single diff tree entry into its component elements.
 108
 109    See git-diff-tree(1) manpage for details about the format of the diff
 110    output. This method returns a dictionary with the following elements:
 111
 112    src_mode - The mode of the source file
 113    dst_mode - The mode of the destination file
 114    src_sha1 - The sha1 for the source file
 115    dst_sha1 - The sha1 fr the destination file
 116    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117    status_score - The score for the status (applicable for 'C' and 'R'
 118                   statuses). This is None if there is no score.
 119    src - The path for the source file.
 120    dst - The path for the destination file. This is only present for
 121          copy or renames. If it is not present, this is None.
 122
 123    If the pattern is not matched, None is returned."""
 124
 125    match = diffTreePattern().next().match(entry)
 126    if match:
 127        return {
 128            'src_mode': match.group(1),
 129            'dst_mode': match.group(2),
 130            'src_sha1': match.group(3),
 131            'dst_sha1': match.group(4),
 132            'status': match.group(5),
 133            'status_score': match.group(6),
 134            'src': match.group(7),
 135            'dst': match.group(10)
 136        }
 137    return None
 138
 139def isModeExec(mode):
 140    # Returns True if the given git mode represents an executable file,
 141    # otherwise False.
 142    return mode[-3:] == "755"
 143
 144def isModeExecChanged(src_mode, dst_mode):
 145    return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148    cmd = "p4 -G %s" % cmd
 149    if verbose:
 150        sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152    # Use a temporary file to avoid deadlocks without
 153    # subprocess.communicate(), which would put another copy
 154    # of stdout into memory.
 155    stdin_file = None
 156    if stdin is not None:
 157        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158        stdin_file.write(stdin)
 159        stdin_file.flush()
 160        stdin_file.seek(0)
 161
 162    p4 = subprocess.Popen(cmd, shell=True,
 163                          stdin=stdin_file,
 164                          stdout=subprocess.PIPE)
 165
 166    result = []
 167    try:
 168        while True:
 169            entry = marshal.load(p4.stdout)
 170            result.append(entry)
 171    except EOFError:
 172        pass
 173    exitCode = p4.wait()
 174    if exitCode != 0:
 175        entry = {}
 176        entry["p4ExitCode"] = exitCode
 177        result.append(entry)
 178
 179    return result
 180
 181def p4Cmd(cmd):
 182    list = p4CmdList(cmd)
 183    result = {}
 184    for entry in list:
 185        result.update(entry)
 186    return result;
 187
 188def p4Where(depotPath):
 189    if not depotPath.endswith("/"):
 190        depotPath += "/"
 191    output = p4Cmd("where %s..." % depotPath)
 192    if output["code"] == "error":
 193        return ""
 194    clientPath = ""
 195    if "path" in output:
 196        clientPath = output.get("path")
 197    elif "data" in output:
 198        data = output.get("data")
 199        lastSpace = data.rfind(" ")
 200        clientPath = data[lastSpace + 1:]
 201
 202    if clientPath.endswith("..."):
 203        clientPath = clientPath[:-3]
 204    return clientPath
 205
 206def currentGitBranch():
 207    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209def isValidGitDir(path):
 210    if (os.path.exists(path + "/HEAD")
 211        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212        return True;
 213    return False
 214
 215def parseRevision(ref):
 216    return read_pipe("git rev-parse %s" % ref).strip()
 217
 218def extractLogMessageFromGitCommit(commit):
 219    logMessage = ""
 220
 221    ## fixme: title is first line of commit, not 1st paragraph.
 222    foundTitle = False
 223    for log in read_pipe_lines("git cat-file commit %s" % commit):
 224       if not foundTitle:
 225           if len(log) == 1:
 226               foundTitle = True
 227           continue
 228
 229       logMessage += log
 230    return logMessage
 231
 232def extractSettingsGitLog(log):
 233    values = {}
 234    for line in log.split("\n"):
 235        line = line.strip()
 236        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237        if not m:
 238            continue
 239
 240        assignments = m.group(1).split (':')
 241        for a in assignments:
 242            vals = a.split ('=')
 243            key = vals[0].strip()
 244            val = ('='.join (vals[1:])).strip()
 245            if val.endswith ('\"') and val.startswith('"'):
 246                val = val[1:-1]
 247
 248            values[key] = val
 249
 250    paths = values.get("depot-paths")
 251    if not paths:
 252        paths = values.get("depot-path")
 253    if paths:
 254        values['depot-paths'] = paths.split(',')
 255    return values
 256
 257def gitBranchExists(branch):
 258    proc = subprocess.Popen(["git", "rev-parse", branch],
 259                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260    return proc.wait() == 0;
 261
 262def gitConfig(key):
 263    return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265def p4BranchesInGit(branchesAreInRemotes = True):
 266    branches = {}
 267
 268    cmdline = "git rev-parse --symbolic "
 269    if branchesAreInRemotes:
 270        cmdline += " --remotes"
 271    else:
 272        cmdline += " --branches"
 273
 274    for line in read_pipe_lines(cmdline):
 275        line = line.strip()
 276
 277        ## only import to p4/
 278        if not line.startswith('p4/') or line == "p4/HEAD":
 279            continue
 280        branch = line
 281
 282        # strip off p4
 283        branch = re.sub ("^p4/", "", line)
 284
 285        branches[branch] = parseRevision(line)
 286    return branches
 287
 288def findUpstreamBranchPoint(head = "HEAD"):
 289    branches = p4BranchesInGit()
 290    # map from depot-path to branch name
 291    branchByDepotPath = {}
 292    for branch in branches.keys():
 293        tip = branches[branch]
 294        log = extractLogMessageFromGitCommit(tip)
 295        settings = extractSettingsGitLog(log)
 296        if settings.has_key("depot-paths"):
 297            paths = ",".join(settings["depot-paths"])
 298            branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300    settings = None
 301    parent = 0
 302    while parent < 65535:
 303        commit = head + "~%s" % parent
 304        log = extractLogMessageFromGitCommit(commit)
 305        settings = extractSettingsGitLog(log)
 306        if settings.has_key("depot-paths"):
 307            paths = ",".join(settings["depot-paths"])
 308            if branchByDepotPath.has_key(paths):
 309                return [branchByDepotPath[paths], settings]
 310
 311        parent = parent + 1
 312
 313    return ["", settings]
 314
 315def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316    if not silent:
 317        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318               % localRefPrefix)
 319
 320    originPrefix = "origin/p4/"
 321
 322    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323        line = line.strip()
 324        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325            continue
 326
 327        headName = line[len(originPrefix):]
 328        remoteHead = localRefPrefix + headName
 329        originHead = line
 330
 331        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332        if (not original.has_key('depot-paths')
 333            or not original.has_key('change')):
 334            continue
 335
 336        update = False
 337        if not gitBranchExists(remoteHead):
 338            if verbose:
 339                print "creating %s" % remoteHead
 340            update = True
 341        else:
 342            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343            if settings.has_key('change') > 0:
 344                if settings['depot-paths'] == original['depot-paths']:
 345                    originP4Change = int(original['change'])
 346                    p4Change = int(settings['change'])
 347                    if originP4Change > p4Change:
 348                        print ("%s (%s) is newer than %s (%s). "
 349                               "Updating p4 branch from origin."
 350                               % (originHead, originP4Change,
 351                                  remoteHead, p4Change))
 352                        update = True
 353                else:
 354                    print ("Ignoring: %s was imported from %s while "
 355                           "%s was imported from %s"
 356                           % (originHead, ','.join(original['depot-paths']),
 357                              remoteHead, ','.join(settings['depot-paths'])))
 358
 359        if update:
 360            system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362def originP4BranchesExist():
 363        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365def p4ChangesForPaths(depotPaths, changeRange):
 366    assert depotPaths
 367    output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                        for p in depotPaths]))
 369
 370    changes = []
 371    for line in output:
 372        changeNum = line.split(" ")[1]
 373        changes.append(int(changeNum))
 374
 375    changes.sort()
 376    return changes
 377
 378class Command:
 379    def __init__(self):
 380        self.usage = "usage: %prog [options]"
 381        self.needsGit = True
 382
 383class P4Debug(Command):
 384    def __init__(self):
 385        Command.__init__(self)
 386        self.options = [
 387            optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                 default=False),
 389            ]
 390        self.description = "A tool to debug the output of p4 -G."
 391        self.needsGit = False
 392        self.verbose = False
 393
 394    def run(self, args):
 395        j = 0
 396        for output in p4CmdList(" ".join(args)):
 397            print 'Element: %d' % j
 398            j += 1
 399            print output
 400        return True
 401
 402class P4RollBack(Command):
 403    def __init__(self):
 404        Command.__init__(self)
 405        self.options = [
 406            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408        ]
 409        self.description = "A tool to debug the multi-branch import. Don't use :)"
 410        self.verbose = False
 411        self.rollbackLocalBranches = False
 412
 413    def run(self, args):
 414        if len(args) != 1:
 415            return False
 416        maxChange = int(args[0])
 417
 418        if "p4ExitCode" in p4Cmd("changes -m 1"):
 419            die("Problems executing p4");
 420
 421        if self.rollbackLocalBranches:
 422            refPrefix = "refs/heads/"
 423            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424        else:
 425            refPrefix = "refs/remotes/"
 426            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428        for line in lines:
 429            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                line = line.strip()
 431                ref = refPrefix + line
 432                log = extractLogMessageFromGitCommit(ref)
 433                settings = extractSettingsGitLog(log)
 434
 435                depotPaths = settings['depot-paths']
 436                change = settings['change']
 437
 438                changed = False
 439
 440                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                           for p in depotPaths]))) == 0:
 442                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                    continue
 445
 446                while change and int(change) > maxChange:
 447                    changed = True
 448                    if self.verbose:
 449                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                    system("git update-ref %s \"%s^\"" % (ref, ref))
 451                    log = extractLogMessageFromGitCommit(ref)
 452                    settings =  extractSettingsGitLog(log)
 453
 454
 455                    depotPaths = settings['depot-paths']
 456                    change = settings['change']
 457
 458                if changed:
 459                    print "%s rewound to %s" % (ref, change)
 460
 461        return True
 462
 463class P4Submit(Command):
 464    def __init__(self):
 465        Command.__init__(self)
 466        self.options = [
 467                optparse.make_option("--continue", action="store_false", dest="firstTime"),
 468                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 469                optparse.make_option("--origin", dest="origin"),
 470                optparse.make_option("--reset", action="store_true", dest="reset"),
 471                optparse.make_option("--log-substitutions", dest="substFile"),
 472                optparse.make_option("--dry-run", action="store_true"),
 473                optparse.make_option("--direct", dest="directSubmit", action="store_true"),
 474                optparse.make_option("--trust-me-like-a-fool", dest="trustMeLikeAFool", action="store_true"),
 475                optparse.make_option("-M", dest="detectRename", action="store_true"),
 476        ]
 477        self.description = "Submit changes from git to the perforce depot."
 478        self.usage += " [name of git branch to submit into perforce depot]"
 479        self.firstTime = True
 480        self.reset = False
 481        self.interactive = True
 482        self.dryRun = False
 483        self.substFile = ""
 484        self.firstTime = True
 485        self.origin = ""
 486        self.directSubmit = False
 487        self.trustMeLikeAFool = False
 488        self.detectRename = False
 489        self.verbose = False
 490        self.isWindows = (platform.system() == "Windows")
 491
 492        self.logSubstitutions = {}
 493        self.logSubstitutions["<enter description here>"] = "%log%"
 494        self.logSubstitutions["\tDetails:"] = "\tDetails:  %log%"
 495
 496    def check(self):
 497        if len(p4CmdList("opened ...")) > 0:
 498            die("You have files opened with perforce! Close them before starting the sync.")
 499
 500    def start(self):
 501        if len(self.config) > 0 and not self.reset:
 502            die("Cannot start sync. Previous sync config found at %s\n"
 503                "If you want to start submitting again from scratch "
 504                "maybe you want to call git-p4 submit --reset" % self.configFile)
 505
 506        commits = []
 507        if self.directSubmit:
 508            commits.append("0")
 509        else:
 510            for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 511                commits.append(line.strip())
 512            commits.reverse()
 513
 514        self.config["commits"] = commits
 515
 516    def prepareLogMessage(self, template, message):
 517        result = ""
 518
 519        for line in template.split("\n"):
 520            if line.startswith("#"):
 521                result += line + "\n"
 522                continue
 523
 524            substituted = False
 525            for key in self.logSubstitutions.keys():
 526                if line.find(key) != -1:
 527                    value = self.logSubstitutions[key]
 528                    value = value.replace("%log%", message)
 529                    if value != "@remove@":
 530                        result += line.replace(key, value) + "\n"
 531                    substituted = True
 532                    break
 533
 534            if not substituted:
 535                result += line + "\n"
 536
 537        return result
 538
 539    def prepareSubmitTemplate(self):
 540        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 541        template = ""
 542        inFilesSection = False
 543        for line in read_pipe_lines("p4 change -o"):
 544            if inFilesSection:
 545                if line.startswith("\t"):
 546                    # path starts and ends with a tab
 547                    path = line[1:]
 548                    lastTab = path.rfind("\t")
 549                    if lastTab != -1:
 550                        path = path[:lastTab]
 551                        if not path.startswith(self.depotPath):
 552                            continue
 553                else:
 554                    inFilesSection = False
 555            else:
 556                if line.startswith("Files:"):
 557                    inFilesSection = True
 558
 559            template += line
 560
 561        return template
 562
 563    def applyCommit(self, id):
 564        if self.directSubmit:
 565            print "Applying local change in working directory/index"
 566            diff = self.diffStatus
 567        else:
 568            print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 569            diffOpts = ("", "-M")[self.detectRename]
 570            diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 571        filesToAdd = set()
 572        filesToDelete = set()
 573        editedFiles = set()
 574        filesToChangeExecBit = {}
 575        for line in diff:
 576            diff = parseDiffTreeEntry(line)
 577            modifier = diff['status']
 578            path = diff['src']
 579            if modifier == "M":
 580                system("p4 edit \"%s\"" % path)
 581                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 582                    filesToChangeExecBit[path] = diff['dst_mode']
 583                editedFiles.add(path)
 584            elif modifier == "A":
 585                filesToAdd.add(path)
 586                filesToChangeExecBit[path] = diff['dst_mode']
 587                if path in filesToDelete:
 588                    filesToDelete.remove(path)
 589            elif modifier == "D":
 590                filesToDelete.add(path)
 591                if path in filesToAdd:
 592                    filesToAdd.remove(path)
 593            elif modifier == "R":
 594                src, dest = diff['src'], diff['dst']
 595                system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 596                system("p4 edit \"%s\"" % (dest))
 597                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 598                    filesToChangeExecBit[dest] = diff['dst_mode']
 599                os.unlink(dest)
 600                editedFiles.add(dest)
 601                filesToDelete.add(src)
 602            else:
 603                die("unknown modifier %s for %s" % (modifier, path))
 604
 605        if self.directSubmit:
 606            diffcmd = "cat \"%s\"" % self.diffFile
 607        else:
 608            diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 609        patchcmd = diffcmd + " | git apply "
 610        tryPatchCmd = patchcmd + "--check -"
 611        applyPatchCmd = patchcmd + "--check --apply -"
 612
 613        if os.system(tryPatchCmd) != 0:
 614            print "Unfortunately applying the change failed!"
 615            print "What do you want to do?"
 616            response = "x"
 617            while response != "s" and response != "a" and response != "w":
 618                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 619                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 620            if response == "s":
 621                print "Skipping! Good luck with the next patches..."
 622                for f in editedFiles:
 623                    system("p4 revert \"%s\"" % f);
 624                for f in filesToAdd:
 625                    system("rm %s" %f)
 626                return
 627            elif response == "a":
 628                os.system(applyPatchCmd)
 629                if len(filesToAdd) > 0:
 630                    print "You may also want to call p4 add on the following files:"
 631                    print " ".join(filesToAdd)
 632                if len(filesToDelete):
 633                    print "The following files should be scheduled for deletion with p4 delete:"
 634                    print " ".join(filesToDelete)
 635                die("Please resolve and submit the conflict manually and "
 636                    + "continue afterwards with git-p4 submit --continue")
 637            elif response == "w":
 638                system(diffcmd + " > patch.txt")
 639                print "Patch saved to patch.txt in %s !" % self.clientPath
 640                die("Please resolve and submit the conflict manually and "
 641                    "continue afterwards with git-p4 submit --continue")
 642
 643        system(applyPatchCmd)
 644
 645        for f in filesToAdd:
 646            system("p4 add \"%s\"" % f)
 647        for f in filesToDelete:
 648            system("p4 revert \"%s\"" % f)
 649            system("p4 delete \"%s\"" % f)
 650
 651        # Set/clear executable bits
 652        for f in filesToChangeExecBit.keys():
 653            mode = filesToChangeExecBit[f]
 654            setP4ExecBit(f, mode)
 655
 656        logMessage = ""
 657        if not self.directSubmit:
 658            logMessage = extractLogMessageFromGitCommit(id)
 659            logMessage = logMessage.replace("\n", "\n\t")
 660            if self.isWindows:
 661                logMessage = logMessage.replace("\n", "\r\n")
 662            logMessage = logMessage.strip()
 663
 664        template = self.prepareSubmitTemplate()
 665
 666        if self.interactive:
 667            submitTemplate = self.prepareLogMessage(template, logMessage)
 668            diff = read_pipe("p4 diff -du ...")
 669
 670            for newFile in filesToAdd:
 671                diff += "==== new file ====\n"
 672                diff += "--- /dev/null\n"
 673                diff += "+++ %s\n" % newFile
 674                f = open(newFile, "r")
 675                for line in f.readlines():
 676                    diff += "+" + line
 677                f.close()
 678
 679            separatorLine = "######## everything below this line is just the diff #######"
 680            if platform.system() == "Windows":
 681                separatorLine += "\r"
 682            separatorLine += "\n"
 683
 684            response = "e"
 685            if self.trustMeLikeAFool:
 686                response = "y"
 687
 688            firstIteration = True
 689            while response == "e":
 690                if not firstIteration:
 691                    response = raw_input("Do you want to submit this change? [y]es/[e]dit/[n]o/[s]kip ")
 692                firstIteration = False
 693                if response == "e":
 694                    [handle, fileName] = tempfile.mkstemp()
 695                    tmpFile = os.fdopen(handle, "w+")
 696                    tmpFile.write(submitTemplate + separatorLine + diff)
 697                    tmpFile.close()
 698                    defaultEditor = "vi"
 699                    if platform.system() == "Windows":
 700                        defaultEditor = "notepad"
 701                    editor = os.environ.get("EDITOR", defaultEditor);
 702                    system(editor + " " + fileName)
 703                    tmpFile = open(fileName, "rb")
 704                    message = tmpFile.read()
 705                    tmpFile.close()
 706                    os.remove(fileName)
 707                    submitTemplate = message[:message.index(separatorLine)]
 708                    if self.isWindows:
 709                        submitTemplate = submitTemplate.replace("\r\n", "\n")
 710
 711            if response == "y" or response == "yes":
 712               if self.dryRun:
 713                   print submitTemplate
 714                   raw_input("Press return to continue...")
 715               else:
 716                   if self.directSubmit:
 717                       print "Submitting to git first"
 718                       os.chdir(self.oldWorkingDirectory)
 719                       write_pipe("git commit -a -F -", submitTemplate)
 720                       os.chdir(self.clientPath)
 721
 722                   write_pipe("p4 submit -i", submitTemplate)
 723            elif response == "s":
 724                for f in editedFiles:
 725                    system("p4 revert \"%s\"" % f);
 726                for f in filesToAdd:
 727                    system("p4 revert \"%s\"" % f);
 728                    system("rm %s" %f)
 729                for f in filesToDelete:
 730                    system("p4 delete \"%s\"" % f);
 731                return
 732            else:
 733                print "Not submitting!"
 734                self.interactive = False
 735        else:
 736            fileName = "submit.txt"
 737            file = open(fileName, "w+")
 738            file.write(self.prepareLogMessage(template, logMessage))
 739            file.close()
 740            print ("Perforce submit template written as %s. "
 741                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 742                   % (fileName, fileName))
 743
 744    def run(self, args):
 745        if len(args) == 0:
 746            self.master = currentGitBranch()
 747            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 748                die("Detecting current git branch failed!")
 749        elif len(args) == 1:
 750            self.master = args[0]
 751        else:
 752            return False
 753
 754        [upstream, settings] = findUpstreamBranchPoint()
 755        self.depotPath = settings['depot-paths'][0]
 756        if len(self.origin) == 0:
 757            self.origin = upstream
 758
 759        if self.verbose:
 760            print "Origin branch is " + self.origin
 761
 762        if len(self.depotPath) == 0:
 763            print "Internal error: cannot locate perforce depot path from existing branches"
 764            sys.exit(128)
 765
 766        self.clientPath = p4Where(self.depotPath)
 767
 768        if len(self.clientPath) == 0:
 769            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 770            sys.exit(128)
 771
 772        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 773        self.oldWorkingDirectory = os.getcwd()
 774
 775        if self.directSubmit:
 776            self.diffStatus = read_pipe_lines("git diff -r --name-status HEAD")
 777            if len(self.diffStatus) == 0:
 778                print "No changes in working directory to submit."
 779                return True
 780            patch = read_pipe("git diff -p --binary --diff-filter=ACMRTUXB HEAD")
 781            self.diffFile = self.gitdir + "/p4-git-diff"
 782            f = open(self.diffFile, "wb")
 783            f.write(patch)
 784            f.close();
 785
 786        os.chdir(self.clientPath)
 787        print "Syncronizing p4 checkout..."
 788        system("p4 sync ...")
 789
 790        if self.reset:
 791            self.firstTime = True
 792
 793        if len(self.substFile) > 0:
 794            for line in open(self.substFile, "r").readlines():
 795                tokens = line.strip().split("=")
 796                self.logSubstitutions[tokens[0]] = tokens[1]
 797
 798        self.check()
 799        self.configFile = self.gitdir + "/p4-git-sync.cfg"
 800        self.config = shelve.open(self.configFile, writeback=True)
 801
 802        if self.firstTime:
 803            self.start()
 804
 805        commits = self.config.get("commits", [])
 806
 807        while len(commits) > 0:
 808            self.firstTime = False
 809            commit = commits[0]
 810            commits = commits[1:]
 811            self.config["commits"] = commits
 812            self.applyCommit(commit)
 813            if not self.interactive:
 814                break
 815
 816        self.config.close()
 817
 818        if self.directSubmit:
 819            os.remove(self.diffFile)
 820
 821        if len(commits) == 0:
 822            if self.firstTime:
 823                print "No changes found to apply between %s and current HEAD" % self.origin
 824            else:
 825                print "All changes applied!"
 826                os.chdir(self.oldWorkingDirectory)
 827
 828                sync = P4Sync()
 829                sync.run([])
 830
 831                response = raw_input("Do you want to rebase current HEAD from Perforce now using git-p4 rebase? [y]es/[n]o ")
 832                if response == "y" or response == "yes":
 833                    rebase = P4Rebase()
 834                    rebase.rebase()
 835            os.remove(self.configFile)
 836
 837        return True
 838
 839class P4Sync(Command):
 840    def __init__(self):
 841        Command.__init__(self)
 842        self.options = [
 843                optparse.make_option("--branch", dest="branch"),
 844                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 845                optparse.make_option("--changesfile", dest="changesFile"),
 846                optparse.make_option("--silent", dest="silent", action="store_true"),
 847                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 848                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 849                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 850                                     help="Import into refs/heads/ , not refs/remotes"),
 851                optparse.make_option("--max-changes", dest="maxChanges"),
 852                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 853                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import")
 854        ]
 855        self.description = """Imports from Perforce into a git repository.\n
 856    example:
 857    //depot/my/project/ -- to import the current head
 858    //depot/my/project/@all -- to import everything
 859    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 860
 861    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 862
 863        self.usage += " //depot/path[@revRange]"
 864        self.silent = False
 865        self.createdBranches = Set()
 866        self.committedChanges = Set()
 867        self.branch = ""
 868        self.detectBranches = False
 869        self.detectLabels = False
 870        self.changesFile = ""
 871        self.syncWithOrigin = True
 872        self.verbose = False
 873        self.importIntoRemotes = True
 874        self.maxChanges = ""
 875        self.isWindows = (platform.system() == "Windows")
 876        self.keepRepoPath = False
 877        self.depotPaths = None
 878        self.p4BranchesInGit = []
 879
 880        if gitConfig("git-p4.syncFromOrigin") == "false":
 881            self.syncWithOrigin = False
 882
 883    def extractFilesFromCommit(self, commit):
 884        files = []
 885        fnum = 0
 886        while commit.has_key("depotFile%s" % fnum):
 887            path =  commit["depotFile%s" % fnum]
 888
 889            found = [p for p in self.depotPaths
 890                     if path.startswith (p)]
 891            if not found:
 892                fnum = fnum + 1
 893                continue
 894
 895            file = {}
 896            file["path"] = path
 897            file["rev"] = commit["rev%s" % fnum]
 898            file["action"] = commit["action%s" % fnum]
 899            file["type"] = commit["type%s" % fnum]
 900            files.append(file)
 901            fnum = fnum + 1
 902        return files
 903
 904    def stripRepoPath(self, path, prefixes):
 905        if self.keepRepoPath:
 906            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 907
 908        for p in prefixes:
 909            if path.startswith(p):
 910                path = path[len(p):]
 911
 912        return path
 913
 914    def splitFilesIntoBranches(self, commit):
 915        branches = {}
 916        fnum = 0
 917        while commit.has_key("depotFile%s" % fnum):
 918            path =  commit["depotFile%s" % fnum]
 919            found = [p for p in self.depotPaths
 920                     if path.startswith (p)]
 921            if not found:
 922                fnum = fnum + 1
 923                continue
 924
 925            file = {}
 926            file["path"] = path
 927            file["rev"] = commit["rev%s" % fnum]
 928            file["action"] = commit["action%s" % fnum]
 929            file["type"] = commit["type%s" % fnum]
 930            fnum = fnum + 1
 931
 932            relPath = self.stripRepoPath(path, self.depotPaths)
 933
 934            for branch in self.knownBranches.keys():
 935
 936                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 937                if relPath.startswith(branch + "/"):
 938                    if branch not in branches:
 939                        branches[branch] = []
 940                    branches[branch].append(file)
 941                    break
 942
 943        return branches
 944
 945    ## Should move this out, doesn't use SELF.
 946    def readP4Files(self, files):
 947        files = [f for f in files
 948                 if f['action'] != 'delete']
 949
 950        if not files:
 951            return
 952
 953        filedata = p4CmdList('-x - print',
 954                             stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 955                                              for f in files]),
 956                             stdin_mode='w+')
 957        if "p4ExitCode" in filedata[0]:
 958            die("Problems executing p4. Error: [%d]."
 959                % (filedata[0]['p4ExitCode']));
 960
 961        j = 0;
 962        contents = {}
 963        while j < len(filedata):
 964            stat = filedata[j]
 965            j += 1
 966            text = ''
 967            while j < len(filedata) and filedata[j]['code'] in ('text',
 968                                                                'binary'):
 969                text += filedata[j]['data']
 970                j += 1
 971
 972
 973            if not stat.has_key('depotFile'):
 974                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 975                continue
 976
 977            contents[stat['depotFile']] = text
 978
 979        for f in files:
 980            assert not f.has_key('data')
 981            f['data'] = contents[f['path']]
 982
 983    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 984        epoch = details["time"]
 985        author = details["user"]
 986
 987        if self.verbose:
 988            print "commit into %s" % branch
 989
 990        # start with reading files; if that fails, we should not
 991        # create a commit.
 992        new_files = []
 993        for f in files:
 994            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 995                new_files.append (f)
 996            else:
 997                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 998        files = new_files
 999        self.readP4Files(files)
1000
1001
1002
1003
1004        self.gitStream.write("commit %s\n" % branch)
1005#        gitStream.write("mark :%s\n" % details["change"])
1006        self.committedChanges.add(int(details["change"]))
1007        committer = ""
1008        if author not in self.users:
1009            self.getUserMapFromPerforceServer()
1010        if author in self.users:
1011            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1012        else:
1013            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1014
1015        self.gitStream.write("committer %s\n" % committer)
1016
1017        self.gitStream.write("data <<EOT\n")
1018        self.gitStream.write(details["desc"])
1019        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1020                             % (','.join (branchPrefixes), details["change"]))
1021        if len(details['options']) > 0:
1022            self.gitStream.write(": options = %s" % details['options'])
1023        self.gitStream.write("]\nEOT\n\n")
1024
1025        if len(parent) > 0:
1026            if self.verbose:
1027                print "parent %s" % parent
1028            self.gitStream.write("from %s\n" % parent)
1029
1030        for file in files:
1031            if file["type"] == "apple":
1032                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1033                continue
1034
1035            relPath = self.stripRepoPath(file['path'], branchPrefixes)
1036            if file["action"] == "delete":
1037                self.gitStream.write("D %s\n" % relPath)
1038            else:
1039                data = file['data']
1040
1041                mode = "644"
1042                if isP4Exec(file["type"]):
1043                    mode = "755"
1044                elif file["type"] == "symlink":
1045                    mode = "120000"
1046                    # p4 print on a symlink contains "target\n", so strip it off
1047                    data = data[:-1]
1048
1049                if self.isWindows and file["type"].endswith("text"):
1050                    data = data.replace("\r\n", "\n")
1051
1052                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1053                self.gitStream.write("data %s\n" % len(data))
1054                self.gitStream.write(data)
1055                self.gitStream.write("\n")
1056
1057        self.gitStream.write("\n")
1058
1059        change = int(details["change"])
1060
1061        if self.labels.has_key(change):
1062            label = self.labels[change]
1063            labelDetails = label[0]
1064            labelRevisions = label[1]
1065            if self.verbose:
1066                print "Change %s is labelled %s" % (change, labelDetails)
1067
1068            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1069                                                    for p in branchPrefixes]))
1070
1071            if len(files) == len(labelRevisions):
1072
1073                cleanedFiles = {}
1074                for info in files:
1075                    if info["action"] == "delete":
1076                        continue
1077                    cleanedFiles[info["depotFile"]] = info["rev"]
1078
1079                if cleanedFiles == labelRevisions:
1080                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1081                    self.gitStream.write("from %s\n" % branch)
1082
1083                    owner = labelDetails["Owner"]
1084                    tagger = ""
1085                    if author in self.users:
1086                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1087                    else:
1088                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1089                    self.gitStream.write("tagger %s\n" % tagger)
1090                    self.gitStream.write("data <<EOT\n")
1091                    self.gitStream.write(labelDetails["Description"])
1092                    self.gitStream.write("EOT\n\n")
1093
1094                else:
1095                    if not self.silent:
1096                        print ("Tag %s does not match with change %s: files do not match."
1097                               % (labelDetails["label"], change))
1098
1099            else:
1100                if not self.silent:
1101                    print ("Tag %s does not match with change %s: file count is different."
1102                           % (labelDetails["label"], change))
1103
1104    def getUserCacheFilename(self):
1105        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1106        return home + "/.gitp4-usercache.txt"
1107
1108    def getUserMapFromPerforceServer(self):
1109        if self.userMapFromPerforceServer:
1110            return
1111        self.users = {}
1112
1113        for output in p4CmdList("users"):
1114            if not output.has_key("User"):
1115                continue
1116            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1117
1118
1119        s = ''
1120        for (key, val) in self.users.items():
1121            s += "%s\t%s\n" % (key, val)
1122
1123        open(self.getUserCacheFilename(), "wb").write(s)
1124        self.userMapFromPerforceServer = True
1125
1126    def loadUserMapFromCache(self):
1127        self.users = {}
1128        self.userMapFromPerforceServer = False
1129        try:
1130            cache = open(self.getUserCacheFilename(), "rb")
1131            lines = cache.readlines()
1132            cache.close()
1133            for line in lines:
1134                entry = line.strip().split("\t")
1135                self.users[entry[0]] = entry[1]
1136        except IOError:
1137            self.getUserMapFromPerforceServer()
1138
1139    def getLabels(self):
1140        self.labels = {}
1141
1142        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1143        if len(l) > 0 and not self.silent:
1144            print "Finding files belonging to labels in %s" % `self.depotPath`
1145
1146        for output in l:
1147            label = output["label"]
1148            revisions = {}
1149            newestChange = 0
1150            if self.verbose:
1151                print "Querying files for label %s" % label
1152            for file in p4CmdList("files "
1153                                  +  ' '.join (["%s...@%s" % (p, label)
1154                                                for p in self.depotPaths])):
1155                revisions[file["depotFile"]] = file["rev"]
1156                change = int(file["change"])
1157                if change > newestChange:
1158                    newestChange = change
1159
1160            self.labels[newestChange] = [output, revisions]
1161
1162        if self.verbose:
1163            print "Label changes: %s" % self.labels.keys()
1164
1165    def guessProjectName(self):
1166        for p in self.depotPaths:
1167            if p.endswith("/"):
1168                p = p[:-1]
1169            p = p[p.strip().rfind("/") + 1:]
1170            if not p.endswith("/"):
1171               p += "/"
1172            return p
1173
1174    def getBranchMapping(self):
1175        lostAndFoundBranches = set()
1176
1177        for info in p4CmdList("branches"):
1178            details = p4Cmd("branch -o %s" % info["branch"])
1179            viewIdx = 0
1180            while details.has_key("View%s" % viewIdx):
1181                paths = details["View%s" % viewIdx].split(" ")
1182                viewIdx = viewIdx + 1
1183                # require standard //depot/foo/... //depot/bar/... mapping
1184                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1185                    continue
1186                source = paths[0]
1187                destination = paths[1]
1188                ## HACK
1189                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1190                    source = source[len(self.depotPaths[0]):-4]
1191                    destination = destination[len(self.depotPaths[0]):-4]
1192
1193                    if destination in self.knownBranches:
1194                        if not self.silent:
1195                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1196                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1197                        continue
1198
1199                    self.knownBranches[destination] = source
1200
1201                    lostAndFoundBranches.discard(destination)
1202
1203                    if source not in self.knownBranches:
1204                        lostAndFoundBranches.add(source)
1205
1206
1207        for branch in lostAndFoundBranches:
1208            self.knownBranches[branch] = branch
1209
1210    def listExistingP4GitBranches(self):
1211        # branches holds mapping from name to commit
1212        branches = p4BranchesInGit(self.importIntoRemotes)
1213        self.p4BranchesInGit = branches.keys()
1214        for branch in branches.keys():
1215            self.initialParents[self.refPrefix + branch] = branches[branch]
1216
1217    def updateOptionDict(self, d):
1218        option_keys = {}
1219        if self.keepRepoPath:
1220            option_keys['keepRepoPath'] = 1
1221
1222        d["options"] = ' '.join(sorted(option_keys.keys()))
1223
1224    def readOptions(self, d):
1225        self.keepRepoPath = (d.has_key('options')
1226                             and ('keepRepoPath' in d['options']))
1227
1228    def gitRefForBranch(self, branch):
1229        if branch == "main":
1230            return self.refPrefix + "master"
1231
1232        if len(branch) <= 0:
1233            return branch
1234
1235        return self.refPrefix + self.projectName + branch
1236
1237    def gitCommitByP4Change(self, ref, change):
1238        if self.verbose:
1239            print "looking in ref " + ref + " for change %s using bisect..." % change
1240
1241        earliestCommit = ""
1242        latestCommit = parseRevision(ref)
1243
1244        while True:
1245            if self.verbose:
1246                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1247            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1248            if len(next) == 0:
1249                if self.verbose:
1250                    print "argh"
1251                return ""
1252            log = extractLogMessageFromGitCommit(next)
1253            settings = extractSettingsGitLog(log)
1254            currentChange = int(settings['change'])
1255            if self.verbose:
1256                print "current change %s" % currentChange
1257
1258            if currentChange == change:
1259                if self.verbose:
1260                    print "found %s" % next
1261                return next
1262
1263            if currentChange < change:
1264                earliestCommit = "^%s" % next
1265            else:
1266                latestCommit = "%s" % next
1267
1268        return ""
1269
1270    def importNewBranch(self, branch, maxChange):
1271        # make fast-import flush all changes to disk and update the refs using the checkpoint
1272        # command so that we can try to find the branch parent in the git history
1273        self.gitStream.write("checkpoint\n\n");
1274        self.gitStream.flush();
1275        branchPrefix = self.depotPaths[0] + branch + "/"
1276        range = "@1,%s" % maxChange
1277        #print "prefix" + branchPrefix
1278        changes = p4ChangesForPaths([branchPrefix], range)
1279        if len(changes) <= 0:
1280            return False
1281        firstChange = changes[0]
1282        #print "first change in branch: %s" % firstChange
1283        sourceBranch = self.knownBranches[branch]
1284        sourceDepotPath = self.depotPaths[0] + sourceBranch
1285        sourceRef = self.gitRefForBranch(sourceBranch)
1286        #print "source " + sourceBranch
1287
1288        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1289        #print "branch parent: %s" % branchParentChange
1290        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1291        if len(gitParent) > 0:
1292            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1293            #print "parent git commit: %s" % gitParent
1294
1295        self.importChanges(changes)
1296        return True
1297
1298    def importChanges(self, changes):
1299        cnt = 1
1300        for change in changes:
1301            description = p4Cmd("describe %s" % change)
1302            self.updateOptionDict(description)
1303
1304            if not self.silent:
1305                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1306                sys.stdout.flush()
1307            cnt = cnt + 1
1308
1309            try:
1310                if self.detectBranches:
1311                    branches = self.splitFilesIntoBranches(description)
1312                    for branch in branches.keys():
1313                        ## HACK  --hwn
1314                        branchPrefix = self.depotPaths[0] + branch + "/"
1315
1316                        parent = ""
1317
1318                        filesForCommit = branches[branch]
1319
1320                        if self.verbose:
1321                            print "branch is %s" % branch
1322
1323                        self.updatedBranches.add(branch)
1324
1325                        if branch not in self.createdBranches:
1326                            self.createdBranches.add(branch)
1327                            parent = self.knownBranches[branch]
1328                            if parent == branch:
1329                                parent = ""
1330                            else:
1331                                fullBranch = self.projectName + branch
1332                                if fullBranch not in self.p4BranchesInGit:
1333                                    if not self.silent:
1334                                        print("\n    Importing new branch %s" % fullBranch);
1335                                    if self.importNewBranch(branch, change - 1):
1336                                        parent = ""
1337                                        self.p4BranchesInGit.append(fullBranch)
1338                                    if not self.silent:
1339                                        print("\n    Resuming with change %s" % change);
1340
1341                                if self.verbose:
1342                                    print "parent determined through known branches: %s" % parent
1343
1344                        branch = self.gitRefForBranch(branch)
1345                        parent = self.gitRefForBranch(parent)
1346
1347                        if self.verbose:
1348                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1349
1350                        if len(parent) == 0 and branch in self.initialParents:
1351                            parent = self.initialParents[branch]
1352                            del self.initialParents[branch]
1353
1354                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1355                else:
1356                    files = self.extractFilesFromCommit(description)
1357                    self.commit(description, files, self.branch, self.depotPaths,
1358                                self.initialParent)
1359                    self.initialParent = ""
1360            except IOError:
1361                print self.gitError.read()
1362                sys.exit(1)
1363
1364    def importHeadRevision(self, revision):
1365        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1366
1367        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1368        details["desc"] = ("Initial import of %s from the state at revision %s"
1369                           % (' '.join(self.depotPaths), revision))
1370        details["change"] = revision
1371        newestRevision = 0
1372
1373        fileCnt = 0
1374        for info in p4CmdList("files "
1375                              +  ' '.join(["%s...%s"
1376                                           % (p, revision)
1377                                           for p in self.depotPaths])):
1378
1379            if info['code'] == 'error':
1380                sys.stderr.write("p4 returned an error: %s\n"
1381                                 % info['data'])
1382                sys.exit(1)
1383
1384
1385            change = int(info["change"])
1386            if change > newestRevision:
1387                newestRevision = change
1388
1389            if info["action"] == "delete":
1390                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1391                #fileCnt = fileCnt + 1
1392                continue
1393
1394            for prop in ["depotFile", "rev", "action", "type" ]:
1395                details["%s%s" % (prop, fileCnt)] = info[prop]
1396
1397            fileCnt = fileCnt + 1
1398
1399        details["change"] = newestRevision
1400        self.updateOptionDict(details)
1401        try:
1402            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1403        except IOError:
1404            print "IO error with git fast-import. Is your git version recent enough?"
1405            print self.gitError.read()
1406
1407
1408    def run(self, args):
1409        self.depotPaths = []
1410        self.changeRange = ""
1411        self.initialParent = ""
1412        self.previousDepotPaths = []
1413
1414        # map from branch depot path to parent branch
1415        self.knownBranches = {}
1416        self.initialParents = {}
1417        self.hasOrigin = originP4BranchesExist()
1418        if not self.syncWithOrigin:
1419            self.hasOrigin = False
1420
1421        if self.importIntoRemotes:
1422            self.refPrefix = "refs/remotes/p4/"
1423        else:
1424            self.refPrefix = "refs/heads/p4/"
1425
1426        if self.syncWithOrigin and self.hasOrigin:
1427            if not self.silent:
1428                print "Syncing with origin first by calling git fetch origin"
1429            system("git fetch origin")
1430
1431        if len(self.branch) == 0:
1432            self.branch = self.refPrefix + "master"
1433            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1434                system("git update-ref %s refs/heads/p4" % self.branch)
1435                system("git branch -D p4");
1436            # create it /after/ importing, when master exists
1437            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1438                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1439
1440        # TODO: should always look at previous commits,
1441        # merge with previous imports, if possible.
1442        if args == []:
1443            if self.hasOrigin:
1444                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1445            self.listExistingP4GitBranches()
1446
1447            if len(self.p4BranchesInGit) > 1:
1448                if not self.silent:
1449                    print "Importing from/into multiple branches"
1450                self.detectBranches = True
1451
1452            if self.verbose:
1453                print "branches: %s" % self.p4BranchesInGit
1454
1455            p4Change = 0
1456            for branch in self.p4BranchesInGit:
1457                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1458
1459                settings = extractSettingsGitLog(logMsg)
1460
1461                self.readOptions(settings)
1462                if (settings.has_key('depot-paths')
1463                    and settings.has_key ('change')):
1464                    change = int(settings['change']) + 1
1465                    p4Change = max(p4Change, change)
1466
1467                    depotPaths = sorted(settings['depot-paths'])
1468                    if self.previousDepotPaths == []:
1469                        self.previousDepotPaths = depotPaths
1470                    else:
1471                        paths = []
1472                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1473                            for i in range(0, min(len(cur), len(prev))):
1474                                if cur[i] <> prev[i]:
1475                                    i = i - 1
1476                                    break
1477
1478                            paths.append (cur[:i + 1])
1479
1480                        self.previousDepotPaths = paths
1481
1482            if p4Change > 0:
1483                self.depotPaths = sorted(self.previousDepotPaths)
1484                self.changeRange = "@%s,#head" % p4Change
1485                if not self.detectBranches:
1486                    self.initialParent = parseRevision(self.branch)
1487                if not self.silent and not self.detectBranches:
1488                    print "Performing incremental import into %s git branch" % self.branch
1489
1490        if not self.branch.startswith("refs/"):
1491            self.branch = "refs/heads/" + self.branch
1492
1493        if len(args) == 0 and self.depotPaths:
1494            if not self.silent:
1495                print "Depot paths: %s" % ' '.join(self.depotPaths)
1496        else:
1497            if self.depotPaths and self.depotPaths != args:
1498                print ("previous import used depot path %s and now %s was specified. "
1499                       "This doesn't work!" % (' '.join (self.depotPaths),
1500                                               ' '.join (args)))
1501                sys.exit(1)
1502
1503            self.depotPaths = sorted(args)
1504
1505        revision = ""
1506        self.users = {}
1507
1508        newPaths = []
1509        for p in self.depotPaths:
1510            if p.find("@") != -1:
1511                atIdx = p.index("@")
1512                self.changeRange = p[atIdx:]
1513                if self.changeRange == "@all":
1514                    self.changeRange = ""
1515                elif ',' not in self.changeRange:
1516                    revision = self.changeRange
1517                    self.changeRange = ""
1518                p = p[:atIdx]
1519            elif p.find("#") != -1:
1520                hashIdx = p.index("#")
1521                revision = p[hashIdx:]
1522                p = p[:hashIdx]
1523            elif self.previousDepotPaths == []:
1524                revision = "#head"
1525
1526            p = re.sub ("\.\.\.$", "", p)
1527            if not p.endswith("/"):
1528                p += "/"
1529
1530            newPaths.append(p)
1531
1532        self.depotPaths = newPaths
1533
1534
1535        self.loadUserMapFromCache()
1536        self.labels = {}
1537        if self.detectLabels:
1538            self.getLabels();
1539
1540        if self.detectBranches:
1541            ## FIXME - what's a P4 projectName ?
1542            self.projectName = self.guessProjectName()
1543
1544            if not self.hasOrigin:
1545                self.getBranchMapping();
1546            if self.verbose:
1547                print "p4-git branches: %s" % self.p4BranchesInGit
1548                print "initial parents: %s" % self.initialParents
1549            for b in self.p4BranchesInGit:
1550                if b != "master":
1551
1552                    ## FIXME
1553                    b = b[len(self.projectName):]
1554                self.createdBranches.add(b)
1555
1556        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1557
1558        importProcess = subprocess.Popen(["git", "fast-import"],
1559                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1560                                         stderr=subprocess.PIPE);
1561        self.gitOutput = importProcess.stdout
1562        self.gitStream = importProcess.stdin
1563        self.gitError = importProcess.stderr
1564
1565        if revision:
1566            self.importHeadRevision(revision)
1567        else:
1568            changes = []
1569
1570            if len(self.changesFile) > 0:
1571                output = open(self.changesFile).readlines()
1572                changeSet = Set()
1573                for line in output:
1574                    changeSet.add(int(line))
1575
1576                for change in changeSet:
1577                    changes.append(change)
1578
1579                changes.sort()
1580            else:
1581                if self.verbose:
1582                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1583                                                              self.changeRange)
1584                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1585
1586                if len(self.maxChanges) > 0:
1587                    changes = changes[:min(int(self.maxChanges), len(changes))]
1588
1589            if len(changes) == 0:
1590                if not self.silent:
1591                    print "No changes to import!"
1592                return True
1593
1594            if not self.silent and not self.detectBranches:
1595                print "Import destination: %s" % self.branch
1596
1597            self.updatedBranches = set()
1598
1599            self.importChanges(changes)
1600
1601            if not self.silent:
1602                print ""
1603                if len(self.updatedBranches) > 0:
1604                    sys.stdout.write("Updated branches: ")
1605                    for b in self.updatedBranches:
1606                        sys.stdout.write("%s " % b)
1607                    sys.stdout.write("\n")
1608
1609        self.gitStream.close()
1610        if importProcess.wait() != 0:
1611            die("fast-import failed: %s" % self.gitError.read())
1612        self.gitOutput.close()
1613        self.gitError.close()
1614
1615        return True
1616
1617class P4Rebase(Command):
1618    def __init__(self):
1619        Command.__init__(self)
1620        self.options = [ ]
1621        self.description = ("Fetches the latest revision from perforce and "
1622                            + "rebases the current work (branch) against it")
1623        self.verbose = False
1624
1625    def run(self, args):
1626        sync = P4Sync()
1627        sync.run([])
1628
1629        return self.rebase()
1630
1631    def rebase(self):
1632        [upstream, settings] = findUpstreamBranchPoint()
1633        if len(upstream) == 0:
1634            die("Cannot find upstream branchpoint for rebase")
1635
1636        # the branchpoint may be p4/foo~3, so strip off the parent
1637        upstream = re.sub("~[0-9]+$", "", upstream)
1638
1639        print "Rebasing the current branch onto %s" % upstream
1640        oldHead = read_pipe("git rev-parse HEAD").strip()
1641        system("git rebase %s" % upstream)
1642        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1643        return True
1644
1645class P4Clone(P4Sync):
1646    def __init__(self):
1647        P4Sync.__init__(self)
1648        self.description = "Creates a new git repository and imports from Perforce into it"
1649        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1650        self.options.append(
1651            optparse.make_option("--destination", dest="cloneDestination",
1652                                 action='store', default=None,
1653                                 help="where to leave result of the clone"))
1654        self.cloneDestination = None
1655        self.needsGit = False
1656
1657    def defaultDestination(self, args):
1658        ## TODO: use common prefix of args?
1659        depotPath = args[0]
1660        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1661        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1662        depotDir = re.sub(r"\.\.\.$,", "", depotDir)
1663        depotDir = re.sub(r"/$", "", depotDir)
1664        return os.path.split(depotDir)[1]
1665
1666    def run(self, args):
1667        if len(args) < 1:
1668            return False
1669
1670        if self.keepRepoPath and not self.cloneDestination:
1671            sys.stderr.write("Must specify destination for --keep-path\n")
1672            sys.exit(1)
1673
1674        depotPaths = args
1675
1676        if not self.cloneDestination and len(depotPaths) > 1:
1677            self.cloneDestination = depotPaths[-1]
1678            depotPaths = depotPaths[:-1]
1679
1680        for p in depotPaths:
1681            if not p.startswith("//"):
1682                return False
1683
1684        if not self.cloneDestination:
1685            self.cloneDestination = self.defaultDestination(args)
1686
1687        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1688        if not os.path.exists(self.cloneDestination):
1689            os.makedirs(self.cloneDestination)
1690        os.chdir(self.cloneDestination)
1691        system("git init")
1692        self.gitdir = os.getcwd() + "/.git"
1693        if not P4Sync.run(self, depotPaths):
1694            return False
1695        if self.branch != "master":
1696            if gitBranchExists("refs/remotes/p4/master"):
1697                system("git branch master refs/remotes/p4/master")
1698                system("git checkout -f")
1699            else:
1700                print "Could not detect main branch. No checkout/master branch created."
1701
1702        return True
1703
1704class P4Branches(Command):
1705    def __init__(self):
1706        Command.__init__(self)
1707        self.options = [ ]
1708        self.description = ("Shows the git branches that hold imports and their "
1709                            + "corresponding perforce depot paths")
1710        self.verbose = False
1711
1712    def run(self, args):
1713        if originP4BranchesExist():
1714            createOrUpdateBranchesFromOrigin()
1715
1716        cmdline = "git rev-parse --symbolic "
1717        cmdline += " --remotes"
1718
1719        for line in read_pipe_lines(cmdline):
1720            line = line.strip()
1721
1722            if not line.startswith('p4/') or line == "p4/HEAD":
1723                continue
1724            branch = line
1725
1726            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1727            settings = extractSettingsGitLog(log)
1728
1729            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1730        return True
1731
1732class HelpFormatter(optparse.IndentedHelpFormatter):
1733    def __init__(self):
1734        optparse.IndentedHelpFormatter.__init__(self)
1735
1736    def format_description(self, description):
1737        if description:
1738            return description + "\n"
1739        else:
1740            return ""
1741
1742def printUsage(commands):
1743    print "usage: %s <command> [options]" % sys.argv[0]
1744    print ""
1745    print "valid commands: %s" % ", ".join(commands)
1746    print ""
1747    print "Try %s <command> --help for command specific help." % sys.argv[0]
1748    print ""
1749
1750commands = {
1751    "debug" : P4Debug,
1752    "submit" : P4Submit,
1753    "commit" : P4Submit,
1754    "sync" : P4Sync,
1755    "rebase" : P4Rebase,
1756    "clone" : P4Clone,
1757    "rollback" : P4RollBack,
1758    "branches" : P4Branches
1759}
1760
1761
1762def main():
1763    if len(sys.argv[1:]) == 0:
1764        printUsage(commands.keys())
1765        sys.exit(2)
1766
1767    cmd = ""
1768    cmdName = sys.argv[1]
1769    try:
1770        klass = commands[cmdName]
1771        cmd = klass()
1772    except KeyError:
1773        print "unknown command %s" % cmdName
1774        print ""
1775        printUsage(commands.keys())
1776        sys.exit(2)
1777
1778    options = cmd.options
1779    cmd.gitdir = os.environ.get("GIT_DIR", None)
1780
1781    args = sys.argv[2:]
1782
1783    if len(options) > 0:
1784        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1785
1786        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1787                                       options,
1788                                       description = cmd.description,
1789                                       formatter = HelpFormatter())
1790
1791        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1792    global verbose
1793    verbose = cmd.verbose
1794    if cmd.needsGit:
1795        if cmd.gitdir == None:
1796            cmd.gitdir = os.path.abspath(".git")
1797            if not isValidGitDir(cmd.gitdir):
1798                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1799                if os.path.exists(cmd.gitdir):
1800                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1801                    if len(cdup) > 0:
1802                        os.chdir(cdup);
1803
1804        if not isValidGitDir(cmd.gitdir):
1805            if isValidGitDir(cmd.gitdir + "/.git"):
1806                cmd.gitdir += "/.git"
1807            else:
1808                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1809
1810        os.environ["GIT_DIR"] = cmd.gitdir
1811
1812    if not cmd.run(args):
1813        parser.print_help()
1814
1815
1816if __name__ == '__main__':
1817    main()