31c5501d113d9668f6fd74d9c2c1557e7719f712
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19def die(msg):
  20    if verbose:
  21        raise Exception(msg)
  22    else:
  23        sys.stderr.write(msg + "\n")
  24        sys.exit(1)
  25
  26def write_pipe(c, str):
  27    if verbose:
  28        sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30    pipe = os.popen(c, 'w')
  31    val = pipe.write(str)
  32    if pipe.close():
  33        die('Command failed: %s' % c)
  34
  35    return val
  36
  37def read_pipe(c, ignore_error=False):
  38    if verbose:
  39        sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41    pipe = os.popen(c, 'rb')
  42    val = pipe.read()
  43    if pipe.close() and not ignore_error:
  44        die('Command failed: %s' % c)
  45
  46    return val
  47
  48
  49def read_pipe_lines(c):
  50    if verbose:
  51        sys.stderr.write('Reading pipe: %s\n' % c)
  52    ## todo: check return status
  53    pipe = os.popen(c, 'rb')
  54    val = pipe.readlines()
  55    if pipe.close():
  56        die('Command failed: %s' % c)
  57
  58    return val
  59
  60def system(cmd):
  61    if verbose:
  62        sys.stderr.write("executing %s\n" % cmd)
  63    if os.system(cmd) != 0:
  64        die("command failed: %s" % cmd)
  65
  66def isP4Exec(kind):
  67    """Determine if a Perforce 'kind' should have execute permission
  68
  69    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70    or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71    a plus sign, it is also executable"""
  72    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74def setP4ExecBit(file, mode):
  75    # Reopens an already open file and changes the execute bit to match
  76    # the execute bit setting in the passed in mode.
  77
  78    p4Type = "+x"
  79
  80    if not isModeExec(mode):
  81        p4Type = getP4OpenedType(file)
  82        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84        if p4Type[-1] == "+":
  85            p4Type = p4Type[0:-1]
  86
  87    system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89def getP4OpenedType(file):
  90    # Returns the perforce file type for the given file.
  91
  92    result = read_pipe("p4 opened %s" % file)
  93    match = re.match(".*\((.+)\)$", result)
  94    if match:
  95        return match.group(1)
  96    else:
  97        die("Could not determine file type for %s" % file)
  98
  99def diffTreePattern():
 100    # This is a simple generator for the diff tree regex pattern. This could be
 101    # a class variable if this and parseDiffTreeEntry were a part of a class.
 102    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103    while True:
 104        yield pattern
 105
 106def parseDiffTreeEntry(entry):
 107    """Parses a single diff tree entry into its component elements.
 108
 109    See git-diff-tree(1) manpage for details about the format of the diff
 110    output. This method returns a dictionary with the following elements:
 111
 112    src_mode - The mode of the source file
 113    dst_mode - The mode of the destination file
 114    src_sha1 - The sha1 for the source file
 115    dst_sha1 - The sha1 fr the destination file
 116    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117    status_score - The score for the status (applicable for 'C' and 'R'
 118                   statuses). This is None if there is no score.
 119    src - The path for the source file.
 120    dst - The path for the destination file. This is only present for
 121          copy or renames. If it is not present, this is None.
 122
 123    If the pattern is not matched, None is returned."""
 124
 125    match = diffTreePattern().next().match(entry)
 126    if match:
 127        return {
 128            'src_mode': match.group(1),
 129            'dst_mode': match.group(2),
 130            'src_sha1': match.group(3),
 131            'dst_sha1': match.group(4),
 132            'status': match.group(5),
 133            'status_score': match.group(6),
 134            'src': match.group(7),
 135            'dst': match.group(10)
 136        }
 137    return None
 138
 139def isModeExec(mode):
 140    # Returns True if the given git mode represents an executable file,
 141    # otherwise False.
 142    return mode[-3:] == "755"
 143
 144def isModeExecChanged(src_mode, dst_mode):
 145    return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148    cmd = "p4 -G %s" % cmd
 149    if verbose:
 150        sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152    # Use a temporary file to avoid deadlocks without
 153    # subprocess.communicate(), which would put another copy
 154    # of stdout into memory.
 155    stdin_file = None
 156    if stdin is not None:
 157        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158        stdin_file.write(stdin)
 159        stdin_file.flush()
 160        stdin_file.seek(0)
 161
 162    p4 = subprocess.Popen(cmd, shell=True,
 163                          stdin=stdin_file,
 164                          stdout=subprocess.PIPE)
 165
 166    result = []
 167    try:
 168        while True:
 169            entry = marshal.load(p4.stdout)
 170            result.append(entry)
 171    except EOFError:
 172        pass
 173    exitCode = p4.wait()
 174    if exitCode != 0:
 175        entry = {}
 176        entry["p4ExitCode"] = exitCode
 177        result.append(entry)
 178
 179    return result
 180
 181def p4Cmd(cmd):
 182    list = p4CmdList(cmd)
 183    result = {}
 184    for entry in list:
 185        result.update(entry)
 186    return result;
 187
 188def p4Where(depotPath):
 189    if not depotPath.endswith("/"):
 190        depotPath += "/"
 191    output = p4Cmd("where %s..." % depotPath)
 192    if output["code"] == "error":
 193        return ""
 194    clientPath = ""
 195    if "path" in output:
 196        clientPath = output.get("path")
 197    elif "data" in output:
 198        data = output.get("data")
 199        lastSpace = data.rfind(" ")
 200        clientPath = data[lastSpace + 1:]
 201
 202    if clientPath.endswith("..."):
 203        clientPath = clientPath[:-3]
 204    return clientPath
 205
 206def currentGitBranch():
 207    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209def isValidGitDir(path):
 210    if (os.path.exists(path + "/HEAD")
 211        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212        return True;
 213    return False
 214
 215def parseRevision(ref):
 216    return read_pipe("git rev-parse %s" % ref).strip()
 217
 218def extractLogMessageFromGitCommit(commit):
 219    logMessage = ""
 220
 221    ## fixme: title is first line of commit, not 1st paragraph.
 222    foundTitle = False
 223    for log in read_pipe_lines("git cat-file commit %s" % commit):
 224       if not foundTitle:
 225           if len(log) == 1:
 226               foundTitle = True
 227           continue
 228
 229       logMessage += log
 230    return logMessage
 231
 232def extractSettingsGitLog(log):
 233    values = {}
 234    for line in log.split("\n"):
 235        line = line.strip()
 236        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237        if not m:
 238            continue
 239
 240        assignments = m.group(1).split (':')
 241        for a in assignments:
 242            vals = a.split ('=')
 243            key = vals[0].strip()
 244            val = ('='.join (vals[1:])).strip()
 245            if val.endswith ('\"') and val.startswith('"'):
 246                val = val[1:-1]
 247
 248            values[key] = val
 249
 250    paths = values.get("depot-paths")
 251    if not paths:
 252        paths = values.get("depot-path")
 253    if paths:
 254        values['depot-paths'] = paths.split(',')
 255    return values
 256
 257def gitBranchExists(branch):
 258    proc = subprocess.Popen(["git", "rev-parse", branch],
 259                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260    return proc.wait() == 0;
 261
 262def gitConfig(key):
 263    return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265def p4BranchesInGit(branchesAreInRemotes = True):
 266    branches = {}
 267
 268    cmdline = "git rev-parse --symbolic "
 269    if branchesAreInRemotes:
 270        cmdline += " --remotes"
 271    else:
 272        cmdline += " --branches"
 273
 274    for line in read_pipe_lines(cmdline):
 275        line = line.strip()
 276
 277        ## only import to p4/
 278        if not line.startswith('p4/') or line == "p4/HEAD":
 279            continue
 280        branch = line
 281
 282        # strip off p4
 283        branch = re.sub ("^p4/", "", line)
 284
 285        branches[branch] = parseRevision(line)
 286    return branches
 287
 288def findUpstreamBranchPoint(head = "HEAD"):
 289    branches = p4BranchesInGit()
 290    # map from depot-path to branch name
 291    branchByDepotPath = {}
 292    for branch in branches.keys():
 293        tip = branches[branch]
 294        log = extractLogMessageFromGitCommit(tip)
 295        settings = extractSettingsGitLog(log)
 296        if settings.has_key("depot-paths"):
 297            paths = ",".join(settings["depot-paths"])
 298            branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300    settings = None
 301    parent = 0
 302    while parent < 65535:
 303        commit = head + "~%s" % parent
 304        log = extractLogMessageFromGitCommit(commit)
 305        settings = extractSettingsGitLog(log)
 306        if settings.has_key("depot-paths"):
 307            paths = ",".join(settings["depot-paths"])
 308            if branchByDepotPath.has_key(paths):
 309                return [branchByDepotPath[paths], settings]
 310
 311        parent = parent + 1
 312
 313    return ["", settings]
 314
 315def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316    if not silent:
 317        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318               % localRefPrefix)
 319
 320    originPrefix = "origin/p4/"
 321
 322    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323        line = line.strip()
 324        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325            continue
 326
 327        headName = line[len(originPrefix):]
 328        remoteHead = localRefPrefix + headName
 329        originHead = line
 330
 331        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332        if (not original.has_key('depot-paths')
 333            or not original.has_key('change')):
 334            continue
 335
 336        update = False
 337        if not gitBranchExists(remoteHead):
 338            if verbose:
 339                print "creating %s" % remoteHead
 340            update = True
 341        else:
 342            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343            if settings.has_key('change') > 0:
 344                if settings['depot-paths'] == original['depot-paths']:
 345                    originP4Change = int(original['change'])
 346                    p4Change = int(settings['change'])
 347                    if originP4Change > p4Change:
 348                        print ("%s (%s) is newer than %s (%s). "
 349                               "Updating p4 branch from origin."
 350                               % (originHead, originP4Change,
 351                                  remoteHead, p4Change))
 352                        update = True
 353                else:
 354                    print ("Ignoring: %s was imported from %s while "
 355                           "%s was imported from %s"
 356                           % (originHead, ','.join(original['depot-paths']),
 357                              remoteHead, ','.join(settings['depot-paths'])))
 358
 359        if update:
 360            system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362def originP4BranchesExist():
 363        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365def p4ChangesForPaths(depotPaths, changeRange):
 366    assert depotPaths
 367    output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                        for p in depotPaths]))
 369
 370    changes = []
 371    for line in output:
 372        changeNum = line.split(" ")[1]
 373        changes.append(int(changeNum))
 374
 375    changes.sort()
 376    return changes
 377
 378class Command:
 379    def __init__(self):
 380        self.usage = "usage: %prog [options]"
 381        self.needsGit = True
 382
 383class P4Debug(Command):
 384    def __init__(self):
 385        Command.__init__(self)
 386        self.options = [
 387            optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                 default=False),
 389            ]
 390        self.description = "A tool to debug the output of p4 -G."
 391        self.needsGit = False
 392        self.verbose = False
 393
 394    def run(self, args):
 395        j = 0
 396        for output in p4CmdList(" ".join(args)):
 397            print 'Element: %d' % j
 398            j += 1
 399            print output
 400        return True
 401
 402class P4RollBack(Command):
 403    def __init__(self):
 404        Command.__init__(self)
 405        self.options = [
 406            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408        ]
 409        self.description = "A tool to debug the multi-branch import. Don't use :)"
 410        self.verbose = False
 411        self.rollbackLocalBranches = False
 412
 413    def run(self, args):
 414        if len(args) != 1:
 415            return False
 416        maxChange = int(args[0])
 417
 418        if "p4ExitCode" in p4Cmd("changes -m 1"):
 419            die("Problems executing p4");
 420
 421        if self.rollbackLocalBranches:
 422            refPrefix = "refs/heads/"
 423            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424        else:
 425            refPrefix = "refs/remotes/"
 426            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428        for line in lines:
 429            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                line = line.strip()
 431                ref = refPrefix + line
 432                log = extractLogMessageFromGitCommit(ref)
 433                settings = extractSettingsGitLog(log)
 434
 435                depotPaths = settings['depot-paths']
 436                change = settings['change']
 437
 438                changed = False
 439
 440                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                           for p in depotPaths]))) == 0:
 442                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                    continue
 445
 446                while change and int(change) > maxChange:
 447                    changed = True
 448                    if self.verbose:
 449                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                    system("git update-ref %s \"%s^\"" % (ref, ref))
 451                    log = extractLogMessageFromGitCommit(ref)
 452                    settings =  extractSettingsGitLog(log)
 453
 454
 455                    depotPaths = settings['depot-paths']
 456                    change = settings['change']
 457
 458                if changed:
 459                    print "%s rewound to %s" % (ref, change)
 460
 461        return True
 462
 463class P4Submit(Command):
 464    def __init__(self):
 465        Command.__init__(self)
 466        self.options = [
 467                optparse.make_option("--continue", action="store_false", dest="firstTime"),
 468                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 469                optparse.make_option("--origin", dest="origin"),
 470                optparse.make_option("--reset", action="store_true", dest="reset"),
 471                optparse.make_option("--log-substitutions", dest="substFile"),
 472                optparse.make_option("--dry-run", action="store_true"),
 473                optparse.make_option("--direct", dest="directSubmit", action="store_true"),
 474                optparse.make_option("--trust-me-like-a-fool", dest="trustMeLikeAFool", action="store_true"),
 475                optparse.make_option("-M", dest="detectRename", action="store_true"),
 476        ]
 477        self.description = "Submit changes from git to the perforce depot."
 478        self.usage += " [name of git branch to submit into perforce depot]"
 479        self.firstTime = True
 480        self.reset = False
 481        self.interactive = True
 482        self.dryRun = False
 483        self.substFile = ""
 484        self.firstTime = True
 485        self.origin = ""
 486        self.directSubmit = False
 487        self.trustMeLikeAFool = False
 488        self.detectRename = False
 489        self.verbose = False
 490        self.isWindows = (platform.system() == "Windows")
 491
 492        self.logSubstitutions = {}
 493        self.logSubstitutions["<enter description here>"] = "%log%"
 494        self.logSubstitutions["\tDetails:"] = "\tDetails:  %log%"
 495
 496    def check(self):
 497        if len(p4CmdList("opened ...")) > 0:
 498            die("You have files opened with perforce! Close them before starting the sync.")
 499
 500    def start(self):
 501        if len(self.config) > 0 and not self.reset:
 502            die("Cannot start sync. Previous sync config found at %s\n"
 503                "If you want to start submitting again from scratch "
 504                "maybe you want to call git-p4 submit --reset" % self.configFile)
 505
 506        commits = []
 507        if self.directSubmit:
 508            commits.append("0")
 509        else:
 510            for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 511                commits.append(line.strip())
 512            commits.reverse()
 513
 514        self.config["commits"] = commits
 515
 516    def prepareLogMessage(self, template, message):
 517        result = ""
 518
 519        for line in template.split("\n"):
 520            if line.startswith("#"):
 521                result += line + "\n"
 522                continue
 523
 524            substituted = False
 525            for key in self.logSubstitutions.keys():
 526                if line.find(key) != -1:
 527                    value = self.logSubstitutions[key]
 528                    value = value.replace("%log%", message)
 529                    if value != "@remove@":
 530                        result += line.replace(key, value) + "\n"
 531                    substituted = True
 532                    break
 533
 534            if not substituted:
 535                result += line + "\n"
 536
 537        return result
 538
 539    def prepareSubmitTemplate(self):
 540        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 541        template = ""
 542        inFilesSection = False
 543        for line in read_pipe_lines("p4 change -o"):
 544            if inFilesSection:
 545                if line.startswith("\t"):
 546                    # path starts and ends with a tab
 547                    path = line[1:]
 548                    lastTab = path.rfind("\t")
 549                    if lastTab != -1:
 550                        path = path[:lastTab]
 551                        if not path.startswith(self.depotPath):
 552                            continue
 553                else:
 554                    inFilesSection = False
 555            else:
 556                if line.startswith("Files:"):
 557                    inFilesSection = True
 558
 559            template += line
 560
 561        return template
 562
 563    def applyCommit(self, id):
 564        if self.directSubmit:
 565            print "Applying local change in working directory/index"
 566            diff = self.diffStatus
 567        else:
 568            print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 569            diffOpts = ("", "-M")[self.detectRename]
 570            diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 571        filesToAdd = set()
 572        filesToDelete = set()
 573        editedFiles = set()
 574        filesToChangeExecBit = {}
 575        for line in diff:
 576            diff = parseDiffTreeEntry(line)
 577            modifier = diff['status']
 578            path = diff['src']
 579            if modifier == "M":
 580                system("p4 edit \"%s\"" % path)
 581                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 582                    filesToChangeExecBit[path] = diff['dst_mode']
 583                editedFiles.add(path)
 584            elif modifier == "A":
 585                filesToAdd.add(path)
 586                filesToChangeExecBit[path] = diff['dst_mode']
 587                if path in filesToDelete:
 588                    filesToDelete.remove(path)
 589            elif modifier == "D":
 590                filesToDelete.add(path)
 591                if path in filesToAdd:
 592                    filesToAdd.remove(path)
 593            elif modifier == "R":
 594                src, dest = diff['src'], diff['dst']
 595                system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 596                system("p4 edit \"%s\"" % (dest))
 597                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 598                    filesToChangeExecBit[dest] = diff['dst_mode']
 599                os.unlink(dest)
 600                editedFiles.add(dest)
 601                filesToDelete.add(src)
 602            else:
 603                die("unknown modifier %s for %s" % (modifier, path))
 604
 605        if self.directSubmit:
 606            diffcmd = "cat \"%s\"" % self.diffFile
 607        else:
 608            diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 609        patchcmd = diffcmd + " | git apply "
 610        tryPatchCmd = patchcmd + "--check -"
 611        applyPatchCmd = patchcmd + "--check --apply -"
 612
 613        if os.system(tryPatchCmd) != 0:
 614            print "Unfortunately applying the change failed!"
 615            print "What do you want to do?"
 616            response = "x"
 617            while response != "s" and response != "a" and response != "w":
 618                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 619                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 620            if response == "s":
 621                print "Skipping! Good luck with the next patches..."
 622                for f in editedFiles:
 623                    system("p4 revert \"%s\"" % f);
 624                for f in filesToAdd:
 625                    system("rm %s" %f)
 626                return
 627            elif response == "a":
 628                os.system(applyPatchCmd)
 629                if len(filesToAdd) > 0:
 630                    print "You may also want to call p4 add on the following files:"
 631                    print " ".join(filesToAdd)
 632                if len(filesToDelete):
 633                    print "The following files should be scheduled for deletion with p4 delete:"
 634                    print " ".join(filesToDelete)
 635                die("Please resolve and submit the conflict manually and "
 636                    + "continue afterwards with git-p4 submit --continue")
 637            elif response == "w":
 638                system(diffcmd + " > patch.txt")
 639                print "Patch saved to patch.txt in %s !" % self.clientPath
 640                die("Please resolve and submit the conflict manually and "
 641                    "continue afterwards with git-p4 submit --continue")
 642
 643        system(applyPatchCmd)
 644
 645        for f in filesToAdd:
 646            system("p4 add \"%s\"" % f)
 647        for f in filesToDelete:
 648            system("p4 revert \"%s\"" % f)
 649            system("p4 delete \"%s\"" % f)
 650
 651        # Set/clear executable bits
 652        for f in filesToChangeExecBit.keys():
 653            mode = filesToChangeExecBit[f]
 654            setP4ExecBit(f, mode)
 655
 656        logMessage = ""
 657        if not self.directSubmit:
 658            logMessage = extractLogMessageFromGitCommit(id)
 659            logMessage = logMessage.replace("\n", "\n\t")
 660            if self.isWindows:
 661                logMessage = logMessage.replace("\n", "\r\n")
 662            logMessage = logMessage.strip()
 663
 664        template = self.prepareSubmitTemplate()
 665
 666        if self.interactive:
 667            submitTemplate = self.prepareLogMessage(template, logMessage)
 668            diff = read_pipe("p4 diff -du ...")
 669
 670            for newFile in filesToAdd:
 671                diff += "==== new file ====\n"
 672                diff += "--- /dev/null\n"
 673                diff += "+++ %s\n" % newFile
 674                f = open(newFile, "r")
 675                for line in f.readlines():
 676                    diff += "+" + line
 677                f.close()
 678
 679            separatorLine = "######## everything below this line is just the diff #######"
 680            if platform.system() == "Windows":
 681                separatorLine += "\r"
 682            separatorLine += "\n"
 683
 684            response = "e"
 685            if self.trustMeLikeAFool:
 686                response = "y"
 687
 688            firstIteration = True
 689            while response == "e":
 690                if not firstIteration:
 691                    response = raw_input("Do you want to submit this change? [y]es/[e]dit/[n]o/[s]kip ")
 692                firstIteration = False
 693                if response == "e":
 694                    [handle, fileName] = tempfile.mkstemp()
 695                    tmpFile = os.fdopen(handle, "w+")
 696                    tmpFile.write(submitTemplate + separatorLine + diff)
 697                    tmpFile.close()
 698                    defaultEditor = "vi"
 699                    if platform.system() == "Windows":
 700                        defaultEditor = "notepad"
 701                    editor = os.environ.get("EDITOR", defaultEditor);
 702                    system(editor + " " + fileName)
 703                    tmpFile = open(fileName, "rb")
 704                    message = tmpFile.read()
 705                    tmpFile.close()
 706                    os.remove(fileName)
 707                    submitTemplate = message[:message.index(separatorLine)]
 708                    if self.isWindows:
 709                        submitTemplate = submitTemplate.replace("\r\n", "\n")
 710
 711            if response == "y" or response == "yes":
 712               if self.dryRun:
 713                   print submitTemplate
 714                   raw_input("Press return to continue...")
 715               else:
 716                   if self.directSubmit:
 717                       print "Submitting to git first"
 718                       os.chdir(self.oldWorkingDirectory)
 719                       write_pipe("git commit -a -F -", submitTemplate)
 720                       os.chdir(self.clientPath)
 721
 722                   write_pipe("p4 submit -i", submitTemplate)
 723            elif response == "s":
 724                for f in editedFiles:
 725                    system("p4 revert \"%s\"" % f);
 726                for f in filesToAdd:
 727                    system("p4 revert \"%s\"" % f);
 728                    system("rm %s" %f)
 729                for f in filesToDelete:
 730                    system("p4 delete \"%s\"" % f);
 731                return
 732            else:
 733                print "Not submitting!"
 734                self.interactive = False
 735        else:
 736            fileName = "submit.txt"
 737            file = open(fileName, "w+")
 738            file.write(self.prepareLogMessage(template, logMessage))
 739            file.close()
 740            print ("Perforce submit template written as %s. "
 741                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 742                   % (fileName, fileName))
 743
 744    def run(self, args):
 745        if len(args) == 0:
 746            self.master = currentGitBranch()
 747            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 748                die("Detecting current git branch failed!")
 749        elif len(args) == 1:
 750            self.master = args[0]
 751        else:
 752            return False
 753
 754        [upstream, settings] = findUpstreamBranchPoint()
 755        self.depotPath = settings['depot-paths'][0]
 756        if len(self.origin) == 0:
 757            self.origin = upstream
 758
 759        if self.verbose:
 760            print "Origin branch is " + self.origin
 761
 762        if len(self.depotPath) == 0:
 763            print "Internal error: cannot locate perforce depot path from existing branches"
 764            sys.exit(128)
 765
 766        self.clientPath = p4Where(self.depotPath)
 767
 768        if len(self.clientPath) == 0:
 769            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 770            sys.exit(128)
 771
 772        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 773        self.oldWorkingDirectory = os.getcwd()
 774
 775        if self.directSubmit:
 776            self.diffStatus = read_pipe_lines("git diff -r --name-status HEAD")
 777            if len(self.diffStatus) == 0:
 778                print "No changes in working directory to submit."
 779                return True
 780            patch = read_pipe("git diff -p --binary --diff-filter=ACMRTUXB HEAD")
 781            self.diffFile = self.gitdir + "/p4-git-diff"
 782            f = open(self.diffFile, "wb")
 783            f.write(patch)
 784            f.close();
 785
 786        os.chdir(self.clientPath)
 787        print "Syncronizing p4 checkout..."
 788        system("p4 sync ...")
 789
 790        if self.reset:
 791            self.firstTime = True
 792
 793        if len(self.substFile) > 0:
 794            for line in open(self.substFile, "r").readlines():
 795                tokens = line.strip().split("=")
 796                self.logSubstitutions[tokens[0]] = tokens[1]
 797
 798        self.check()
 799        self.configFile = self.gitdir + "/p4-git-sync.cfg"
 800        self.config = shelve.open(self.configFile, writeback=True)
 801
 802        if self.firstTime:
 803            self.start()
 804
 805        commits = self.config.get("commits", [])
 806
 807        while len(commits) > 0:
 808            self.firstTime = False
 809            commit = commits[0]
 810            commits = commits[1:]
 811            self.config["commits"] = commits
 812            self.applyCommit(commit)
 813            if not self.interactive:
 814                break
 815
 816        self.config.close()
 817
 818        if self.directSubmit:
 819            os.remove(self.diffFile)
 820
 821        if len(commits) == 0:
 822            if self.firstTime:
 823                print "No changes found to apply between %s and current HEAD" % self.origin
 824            else:
 825                print "All changes applied!"
 826                os.chdir(self.oldWorkingDirectory)
 827
 828                sync = P4Sync()
 829                sync.run([])
 830
 831                response = raw_input("Do you want to rebase current HEAD from Perforce now using git-p4 rebase? [y]es/[n]o ")
 832                if response == "y" or response == "yes":
 833                    rebase = P4Rebase()
 834                    rebase.rebase()
 835            os.remove(self.configFile)
 836
 837        return True
 838
 839class P4Sync(Command):
 840    def __init__(self):
 841        Command.__init__(self)
 842        self.options = [
 843                optparse.make_option("--branch", dest="branch"),
 844                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 845                optparse.make_option("--changesfile", dest="changesFile"),
 846                optparse.make_option("--silent", dest="silent", action="store_true"),
 847                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 848                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 849                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 850                                     help="Import into refs/heads/ , not refs/remotes"),
 851                optparse.make_option("--max-changes", dest="maxChanges"),
 852                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 853                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import")
 854        ]
 855        self.description = """Imports from Perforce into a git repository.\n
 856    example:
 857    //depot/my/project/ -- to import the current head
 858    //depot/my/project/@all -- to import everything
 859    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 860
 861    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 862
 863        self.usage += " //depot/path[@revRange]"
 864        self.silent = False
 865        self.createdBranches = Set()
 866        self.committedChanges = Set()
 867        self.branch = ""
 868        self.detectBranches = False
 869        self.detectLabels = False
 870        self.changesFile = ""
 871        self.syncWithOrigin = True
 872        self.verbose = False
 873        self.importIntoRemotes = True
 874        self.maxChanges = ""
 875        self.isWindows = (platform.system() == "Windows")
 876        self.keepRepoPath = False
 877        self.depotPaths = None
 878        self.p4BranchesInGit = []
 879
 880        if gitConfig("git-p4.syncFromOrigin") == "false":
 881            self.syncWithOrigin = False
 882
 883    def extractFilesFromCommit(self, commit):
 884        files = []
 885        fnum = 0
 886        while commit.has_key("depotFile%s" % fnum):
 887            path =  commit["depotFile%s" % fnum]
 888
 889            found = [p for p in self.depotPaths
 890                     if path.startswith (p)]
 891            if not found:
 892                fnum = fnum + 1
 893                continue
 894
 895            file = {}
 896            file["path"] = path
 897            file["rev"] = commit["rev%s" % fnum]
 898            file["action"] = commit["action%s" % fnum]
 899            file["type"] = commit["type%s" % fnum]
 900            files.append(file)
 901            fnum = fnum + 1
 902        return files
 903
 904    def stripRepoPath(self, path, prefixes):
 905        if self.keepRepoPath:
 906            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 907
 908        for p in prefixes:
 909            if path.startswith(p):
 910                path = path[len(p):]
 911
 912        return path
 913
 914    def splitFilesIntoBranches(self, commit):
 915        branches = {}
 916        fnum = 0
 917        while commit.has_key("depotFile%s" % fnum):
 918            path =  commit["depotFile%s" % fnum]
 919            found = [p for p in self.depotPaths
 920                     if path.startswith (p)]
 921            if not found:
 922                fnum = fnum + 1
 923                continue
 924
 925            file = {}
 926            file["path"] = path
 927            file["rev"] = commit["rev%s" % fnum]
 928            file["action"] = commit["action%s" % fnum]
 929            file["type"] = commit["type%s" % fnum]
 930            fnum = fnum + 1
 931
 932            relPath = self.stripRepoPath(path, self.depotPaths)
 933
 934            for branch in self.knownBranches.keys():
 935
 936                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 937                if relPath.startswith(branch + "/"):
 938                    if branch not in branches:
 939                        branches[branch] = []
 940                    branches[branch].append(file)
 941                    break
 942
 943        return branches
 944
 945    ## Should move this out, doesn't use SELF.
 946    def readP4Files(self, files):
 947        files = [f for f in files
 948                 if f['action'] != 'delete']
 949
 950        if not files:
 951            return
 952
 953        filedata = p4CmdList('-x - print',
 954                             stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 955                                              for f in files]),
 956                             stdin_mode='w+')
 957        if "p4ExitCode" in filedata[0]:
 958            die("Problems executing p4. Error: [%d]."
 959                % (filedata[0]['p4ExitCode']));
 960
 961        j = 0;
 962        contents = {}
 963        while j < len(filedata):
 964            stat = filedata[j]
 965            j += 1
 966            text = ''
 967            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 968                tmp = filedata[j]['data']
 969                if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 970                    tmp = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', tmp)
 971                elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 972                    tmp = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', tmp)
 973                text += tmp
 974                j += 1
 975
 976
 977            if not stat.has_key('depotFile'):
 978                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 979                continue
 980
 981            contents[stat['depotFile']] = text
 982
 983        for f in files:
 984            assert not f.has_key('data')
 985            f['data'] = contents[f['path']]
 986
 987    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 988        epoch = details["time"]
 989        author = details["user"]
 990
 991        if self.verbose:
 992            print "commit into %s" % branch
 993
 994        # start with reading files; if that fails, we should not
 995        # create a commit.
 996        new_files = []
 997        for f in files:
 998            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 999                new_files.append (f)
1000            else:
1001                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1002        files = new_files
1003        self.readP4Files(files)
1004
1005
1006
1007
1008        self.gitStream.write("commit %s\n" % branch)
1009#        gitStream.write("mark :%s\n" % details["change"])
1010        self.committedChanges.add(int(details["change"]))
1011        committer = ""
1012        if author not in self.users:
1013            self.getUserMapFromPerforceServer()
1014        if author in self.users:
1015            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1016        else:
1017            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1018
1019        self.gitStream.write("committer %s\n" % committer)
1020
1021        self.gitStream.write("data <<EOT\n")
1022        self.gitStream.write(details["desc"])
1023        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1024                             % (','.join (branchPrefixes), details["change"]))
1025        if len(details['options']) > 0:
1026            self.gitStream.write(": options = %s" % details['options'])
1027        self.gitStream.write("]\nEOT\n\n")
1028
1029        if len(parent) > 0:
1030            if self.verbose:
1031                print "parent %s" % parent
1032            self.gitStream.write("from %s\n" % parent)
1033
1034        for file in files:
1035            if file["type"] == "apple":
1036                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1037                continue
1038
1039            relPath = self.stripRepoPath(file['path'], branchPrefixes)
1040            if file["action"] == "delete":
1041                self.gitStream.write("D %s\n" % relPath)
1042            else:
1043                data = file['data']
1044
1045                mode = "644"
1046                if isP4Exec(file["type"]):
1047                    mode = "755"
1048                elif file["type"] == "symlink":
1049                    mode = "120000"
1050                    # p4 print on a symlink contains "target\n", so strip it off
1051                    data = data[:-1]
1052
1053                if self.isWindows and file["type"].endswith("text"):
1054                    data = data.replace("\r\n", "\n")
1055
1056                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1057                self.gitStream.write("data %s\n" % len(data))
1058                self.gitStream.write(data)
1059                self.gitStream.write("\n")
1060
1061        self.gitStream.write("\n")
1062
1063        change = int(details["change"])
1064
1065        if self.labels.has_key(change):
1066            label = self.labels[change]
1067            labelDetails = label[0]
1068            labelRevisions = label[1]
1069            if self.verbose:
1070                print "Change %s is labelled %s" % (change, labelDetails)
1071
1072            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1073                                                    for p in branchPrefixes]))
1074
1075            if len(files) == len(labelRevisions):
1076
1077                cleanedFiles = {}
1078                for info in files:
1079                    if info["action"] == "delete":
1080                        continue
1081                    cleanedFiles[info["depotFile"]] = info["rev"]
1082
1083                if cleanedFiles == labelRevisions:
1084                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1085                    self.gitStream.write("from %s\n" % branch)
1086
1087                    owner = labelDetails["Owner"]
1088                    tagger = ""
1089                    if author in self.users:
1090                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1091                    else:
1092                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1093                    self.gitStream.write("tagger %s\n" % tagger)
1094                    self.gitStream.write("data <<EOT\n")
1095                    self.gitStream.write(labelDetails["Description"])
1096                    self.gitStream.write("EOT\n\n")
1097
1098                else:
1099                    if not self.silent:
1100                        print ("Tag %s does not match with change %s: files do not match."
1101                               % (labelDetails["label"], change))
1102
1103            else:
1104                if not self.silent:
1105                    print ("Tag %s does not match with change %s: file count is different."
1106                           % (labelDetails["label"], change))
1107
1108    def getUserCacheFilename(self):
1109        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1110        return home + "/.gitp4-usercache.txt"
1111
1112    def getUserMapFromPerforceServer(self):
1113        if self.userMapFromPerforceServer:
1114            return
1115        self.users = {}
1116
1117        for output in p4CmdList("users"):
1118            if not output.has_key("User"):
1119                continue
1120            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1121
1122
1123        s = ''
1124        for (key, val) in self.users.items():
1125            s += "%s\t%s\n" % (key, val)
1126
1127        open(self.getUserCacheFilename(), "wb").write(s)
1128        self.userMapFromPerforceServer = True
1129
1130    def loadUserMapFromCache(self):
1131        self.users = {}
1132        self.userMapFromPerforceServer = False
1133        try:
1134            cache = open(self.getUserCacheFilename(), "rb")
1135            lines = cache.readlines()
1136            cache.close()
1137            for line in lines:
1138                entry = line.strip().split("\t")
1139                self.users[entry[0]] = entry[1]
1140        except IOError:
1141            self.getUserMapFromPerforceServer()
1142
1143    def getLabels(self):
1144        self.labels = {}
1145
1146        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1147        if len(l) > 0 and not self.silent:
1148            print "Finding files belonging to labels in %s" % `self.depotPaths`
1149
1150        for output in l:
1151            label = output["label"]
1152            revisions = {}
1153            newestChange = 0
1154            if self.verbose:
1155                print "Querying files for label %s" % label
1156            for file in p4CmdList("files "
1157                                  +  ' '.join (["%s...@%s" % (p, label)
1158                                                for p in self.depotPaths])):
1159                revisions[file["depotFile"]] = file["rev"]
1160                change = int(file["change"])
1161                if change > newestChange:
1162                    newestChange = change
1163
1164            self.labels[newestChange] = [output, revisions]
1165
1166        if self.verbose:
1167            print "Label changes: %s" % self.labels.keys()
1168
1169    def guessProjectName(self):
1170        for p in self.depotPaths:
1171            if p.endswith("/"):
1172                p = p[:-1]
1173            p = p[p.strip().rfind("/") + 1:]
1174            if not p.endswith("/"):
1175               p += "/"
1176            return p
1177
1178    def getBranchMapping(self):
1179        lostAndFoundBranches = set()
1180
1181        for info in p4CmdList("branches"):
1182            details = p4Cmd("branch -o %s" % info["branch"])
1183            viewIdx = 0
1184            while details.has_key("View%s" % viewIdx):
1185                paths = details["View%s" % viewIdx].split(" ")
1186                viewIdx = viewIdx + 1
1187                # require standard //depot/foo/... //depot/bar/... mapping
1188                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1189                    continue
1190                source = paths[0]
1191                destination = paths[1]
1192                ## HACK
1193                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1194                    source = source[len(self.depotPaths[0]):-4]
1195                    destination = destination[len(self.depotPaths[0]):-4]
1196
1197                    if destination in self.knownBranches:
1198                        if not self.silent:
1199                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1200                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1201                        continue
1202
1203                    self.knownBranches[destination] = source
1204
1205                    lostAndFoundBranches.discard(destination)
1206
1207                    if source not in self.knownBranches:
1208                        lostAndFoundBranches.add(source)
1209
1210
1211        for branch in lostAndFoundBranches:
1212            self.knownBranches[branch] = branch
1213
1214    def getBranchMappingFromGitBranches(self):
1215        branches = p4BranchesInGit(self.importIntoRemotes)
1216        for branch in branches.keys():
1217            if branch == "master":
1218                branch = "main"
1219            else:
1220                branch = branch[len(self.projectName):]
1221            self.knownBranches[branch] = branch
1222
1223    def listExistingP4GitBranches(self):
1224        # branches holds mapping from name to commit
1225        branches = p4BranchesInGit(self.importIntoRemotes)
1226        self.p4BranchesInGit = branches.keys()
1227        for branch in branches.keys():
1228            self.initialParents[self.refPrefix + branch] = branches[branch]
1229
1230    def updateOptionDict(self, d):
1231        option_keys = {}
1232        if self.keepRepoPath:
1233            option_keys['keepRepoPath'] = 1
1234
1235        d["options"] = ' '.join(sorted(option_keys.keys()))
1236
1237    def readOptions(self, d):
1238        self.keepRepoPath = (d.has_key('options')
1239                             and ('keepRepoPath' in d['options']))
1240
1241    def gitRefForBranch(self, branch):
1242        if branch == "main":
1243            return self.refPrefix + "master"
1244
1245        if len(branch) <= 0:
1246            return branch
1247
1248        return self.refPrefix + self.projectName + branch
1249
1250    def gitCommitByP4Change(self, ref, change):
1251        if self.verbose:
1252            print "looking in ref " + ref + " for change %s using bisect..." % change
1253
1254        earliestCommit = ""
1255        latestCommit = parseRevision(ref)
1256
1257        while True:
1258            if self.verbose:
1259                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1260            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1261            if len(next) == 0:
1262                if self.verbose:
1263                    print "argh"
1264                return ""
1265            log = extractLogMessageFromGitCommit(next)
1266            settings = extractSettingsGitLog(log)
1267            currentChange = int(settings['change'])
1268            if self.verbose:
1269                print "current change %s" % currentChange
1270
1271            if currentChange == change:
1272                if self.verbose:
1273                    print "found %s" % next
1274                return next
1275
1276            if currentChange < change:
1277                earliestCommit = "^%s" % next
1278            else:
1279                latestCommit = "%s" % next
1280
1281        return ""
1282
1283    def importNewBranch(self, branch, maxChange):
1284        # make fast-import flush all changes to disk and update the refs using the checkpoint
1285        # command so that we can try to find the branch parent in the git history
1286        self.gitStream.write("checkpoint\n\n");
1287        self.gitStream.flush();
1288        branchPrefix = self.depotPaths[0] + branch + "/"
1289        range = "@1,%s" % maxChange
1290        #print "prefix" + branchPrefix
1291        changes = p4ChangesForPaths([branchPrefix], range)
1292        if len(changes) <= 0:
1293            return False
1294        firstChange = changes[0]
1295        #print "first change in branch: %s" % firstChange
1296        sourceBranch = self.knownBranches[branch]
1297        sourceDepotPath = self.depotPaths[0] + sourceBranch
1298        sourceRef = self.gitRefForBranch(sourceBranch)
1299        #print "source " + sourceBranch
1300
1301        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1302        #print "branch parent: %s" % branchParentChange
1303        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1304        if len(gitParent) > 0:
1305            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1306            #print "parent git commit: %s" % gitParent
1307
1308        self.importChanges(changes)
1309        return True
1310
1311    def importChanges(self, changes):
1312        cnt = 1
1313        for change in changes:
1314            description = p4Cmd("describe %s" % change)
1315            self.updateOptionDict(description)
1316
1317            if not self.silent:
1318                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1319                sys.stdout.flush()
1320            cnt = cnt + 1
1321
1322            try:
1323                if self.detectBranches:
1324                    branches = self.splitFilesIntoBranches(description)
1325                    for branch in branches.keys():
1326                        ## HACK  --hwn
1327                        branchPrefix = self.depotPaths[0] + branch + "/"
1328
1329                        parent = ""
1330
1331                        filesForCommit = branches[branch]
1332
1333                        if self.verbose:
1334                            print "branch is %s" % branch
1335
1336                        self.updatedBranches.add(branch)
1337
1338                        if branch not in self.createdBranches:
1339                            self.createdBranches.add(branch)
1340                            parent = self.knownBranches[branch]
1341                            if parent == branch:
1342                                parent = ""
1343                            else:
1344                                fullBranch = self.projectName + branch
1345                                if fullBranch not in self.p4BranchesInGit:
1346                                    if not self.silent:
1347                                        print("\n    Importing new branch %s" % fullBranch);
1348                                    if self.importNewBranch(branch, change - 1):
1349                                        parent = ""
1350                                        self.p4BranchesInGit.append(fullBranch)
1351                                    if not self.silent:
1352                                        print("\n    Resuming with change %s" % change);
1353
1354                                if self.verbose:
1355                                    print "parent determined through known branches: %s" % parent
1356
1357                        branch = self.gitRefForBranch(branch)
1358                        parent = self.gitRefForBranch(parent)
1359
1360                        if self.verbose:
1361                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1362
1363                        if len(parent) == 0 and branch in self.initialParents:
1364                            parent = self.initialParents[branch]
1365                            del self.initialParents[branch]
1366
1367                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1368                else:
1369                    files = self.extractFilesFromCommit(description)
1370                    self.commit(description, files, self.branch, self.depotPaths,
1371                                self.initialParent)
1372                    self.initialParent = ""
1373            except IOError:
1374                print self.gitError.read()
1375                sys.exit(1)
1376
1377    def importHeadRevision(self, revision):
1378        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1379
1380        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1381        details["desc"] = ("Initial import of %s from the state at revision %s"
1382                           % (' '.join(self.depotPaths), revision))
1383        details["change"] = revision
1384        newestRevision = 0
1385
1386        fileCnt = 0
1387        for info in p4CmdList("files "
1388                              +  ' '.join(["%s...%s"
1389                                           % (p, revision)
1390                                           for p in self.depotPaths])):
1391
1392            if info['code'] == 'error':
1393                sys.stderr.write("p4 returned an error: %s\n"
1394                                 % info['data'])
1395                sys.exit(1)
1396
1397
1398            change = int(info["change"])
1399            if change > newestRevision:
1400                newestRevision = change
1401
1402            if info["action"] == "delete":
1403                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1404                #fileCnt = fileCnt + 1
1405                continue
1406
1407            for prop in ["depotFile", "rev", "action", "type" ]:
1408                details["%s%s" % (prop, fileCnt)] = info[prop]
1409
1410            fileCnt = fileCnt + 1
1411
1412        details["change"] = newestRevision
1413        self.updateOptionDict(details)
1414        try:
1415            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1416        except IOError:
1417            print "IO error with git fast-import. Is your git version recent enough?"
1418            print self.gitError.read()
1419
1420
1421    def run(self, args):
1422        self.depotPaths = []
1423        self.changeRange = ""
1424        self.initialParent = ""
1425        self.previousDepotPaths = []
1426
1427        # map from branch depot path to parent branch
1428        self.knownBranches = {}
1429        self.initialParents = {}
1430        self.hasOrigin = originP4BranchesExist()
1431        if not self.syncWithOrigin:
1432            self.hasOrigin = False
1433
1434        if self.importIntoRemotes:
1435            self.refPrefix = "refs/remotes/p4/"
1436        else:
1437            self.refPrefix = "refs/heads/p4/"
1438
1439        if self.syncWithOrigin and self.hasOrigin:
1440            if not self.silent:
1441                print "Syncing with origin first by calling git fetch origin"
1442            system("git fetch origin")
1443
1444        if len(self.branch) == 0:
1445            self.branch = self.refPrefix + "master"
1446            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1447                system("git update-ref %s refs/heads/p4" % self.branch)
1448                system("git branch -D p4");
1449            # create it /after/ importing, when master exists
1450            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1451                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1452
1453        # TODO: should always look at previous commits,
1454        # merge with previous imports, if possible.
1455        if args == []:
1456            if self.hasOrigin:
1457                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1458            self.listExistingP4GitBranches()
1459
1460            if len(self.p4BranchesInGit) > 1:
1461                if not self.silent:
1462                    print "Importing from/into multiple branches"
1463                self.detectBranches = True
1464
1465            if self.verbose:
1466                print "branches: %s" % self.p4BranchesInGit
1467
1468            p4Change = 0
1469            for branch in self.p4BranchesInGit:
1470                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1471
1472                settings = extractSettingsGitLog(logMsg)
1473
1474                self.readOptions(settings)
1475                if (settings.has_key('depot-paths')
1476                    and settings.has_key ('change')):
1477                    change = int(settings['change']) + 1
1478                    p4Change = max(p4Change, change)
1479
1480                    depotPaths = sorted(settings['depot-paths'])
1481                    if self.previousDepotPaths == []:
1482                        self.previousDepotPaths = depotPaths
1483                    else:
1484                        paths = []
1485                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1486                            for i in range(0, min(len(cur), len(prev))):
1487                                if cur[i] <> prev[i]:
1488                                    i = i - 1
1489                                    break
1490
1491                            paths.append (cur[:i + 1])
1492
1493                        self.previousDepotPaths = paths
1494
1495            if p4Change > 0:
1496                self.depotPaths = sorted(self.previousDepotPaths)
1497                self.changeRange = "@%s,#head" % p4Change
1498                if not self.detectBranches:
1499                    self.initialParent = parseRevision(self.branch)
1500                if not self.silent and not self.detectBranches:
1501                    print "Performing incremental import into %s git branch" % self.branch
1502
1503        if not self.branch.startswith("refs/"):
1504            self.branch = "refs/heads/" + self.branch
1505
1506        if len(args) == 0 and self.depotPaths:
1507            if not self.silent:
1508                print "Depot paths: %s" % ' '.join(self.depotPaths)
1509        else:
1510            if self.depotPaths and self.depotPaths != args:
1511                print ("previous import used depot path %s and now %s was specified. "
1512                       "This doesn't work!" % (' '.join (self.depotPaths),
1513                                               ' '.join (args)))
1514                sys.exit(1)
1515
1516            self.depotPaths = sorted(args)
1517
1518        revision = ""
1519        self.users = {}
1520
1521        newPaths = []
1522        for p in self.depotPaths:
1523            if p.find("@") != -1:
1524                atIdx = p.index("@")
1525                self.changeRange = p[atIdx:]
1526                if self.changeRange == "@all":
1527                    self.changeRange = ""
1528                elif ',' not in self.changeRange:
1529                    revision = self.changeRange
1530                    self.changeRange = ""
1531                p = p[:atIdx]
1532            elif p.find("#") != -1:
1533                hashIdx = p.index("#")
1534                revision = p[hashIdx:]
1535                p = p[:hashIdx]
1536            elif self.previousDepotPaths == []:
1537                revision = "#head"
1538
1539            p = re.sub ("\.\.\.$", "", p)
1540            if not p.endswith("/"):
1541                p += "/"
1542
1543            newPaths.append(p)
1544
1545        self.depotPaths = newPaths
1546
1547
1548        self.loadUserMapFromCache()
1549        self.labels = {}
1550        if self.detectLabels:
1551            self.getLabels();
1552
1553        if self.detectBranches:
1554            ## FIXME - what's a P4 projectName ?
1555            self.projectName = self.guessProjectName()
1556
1557            if self.hasOrigin:
1558                self.getBranchMappingFromGitBranches()
1559            else:
1560                self.getBranchMapping()
1561            if self.verbose:
1562                print "p4-git branches: %s" % self.p4BranchesInGit
1563                print "initial parents: %s" % self.initialParents
1564            for b in self.p4BranchesInGit:
1565                if b != "master":
1566
1567                    ## FIXME
1568                    b = b[len(self.projectName):]
1569                self.createdBranches.add(b)
1570
1571        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1572
1573        importProcess = subprocess.Popen(["git", "fast-import"],
1574                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1575                                         stderr=subprocess.PIPE);
1576        self.gitOutput = importProcess.stdout
1577        self.gitStream = importProcess.stdin
1578        self.gitError = importProcess.stderr
1579
1580        if revision:
1581            self.importHeadRevision(revision)
1582        else:
1583            changes = []
1584
1585            if len(self.changesFile) > 0:
1586                output = open(self.changesFile).readlines()
1587                changeSet = Set()
1588                for line in output:
1589                    changeSet.add(int(line))
1590
1591                for change in changeSet:
1592                    changes.append(change)
1593
1594                changes.sort()
1595            else:
1596                if self.verbose:
1597                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1598                                                              self.changeRange)
1599                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1600
1601                if len(self.maxChanges) > 0:
1602                    changes = changes[:min(int(self.maxChanges), len(changes))]
1603
1604            if len(changes) == 0:
1605                if not self.silent:
1606                    print "No changes to import!"
1607                return True
1608
1609            if not self.silent and not self.detectBranches:
1610                print "Import destination: %s" % self.branch
1611
1612            self.updatedBranches = set()
1613
1614            self.importChanges(changes)
1615
1616            if not self.silent:
1617                print ""
1618                if len(self.updatedBranches) > 0:
1619                    sys.stdout.write("Updated branches: ")
1620                    for b in self.updatedBranches:
1621                        sys.stdout.write("%s " % b)
1622                    sys.stdout.write("\n")
1623
1624        self.gitStream.close()
1625        if importProcess.wait() != 0:
1626            die("fast-import failed: %s" % self.gitError.read())
1627        self.gitOutput.close()
1628        self.gitError.close()
1629
1630        return True
1631
1632class P4Rebase(Command):
1633    def __init__(self):
1634        Command.__init__(self)
1635        self.options = [ ]
1636        self.description = ("Fetches the latest revision from perforce and "
1637                            + "rebases the current work (branch) against it")
1638        self.verbose = False
1639
1640    def run(self, args):
1641        sync = P4Sync()
1642        sync.run([])
1643
1644        return self.rebase()
1645
1646    def rebase(self):
1647        [upstream, settings] = findUpstreamBranchPoint()
1648        if len(upstream) == 0:
1649            die("Cannot find upstream branchpoint for rebase")
1650
1651        # the branchpoint may be p4/foo~3, so strip off the parent
1652        upstream = re.sub("~[0-9]+$", "", upstream)
1653
1654        print "Rebasing the current branch onto %s" % upstream
1655        oldHead = read_pipe("git rev-parse HEAD").strip()
1656        system("git rebase %s" % upstream)
1657        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1658        return True
1659
1660class P4Clone(P4Sync):
1661    def __init__(self):
1662        P4Sync.__init__(self)
1663        self.description = "Creates a new git repository and imports from Perforce into it"
1664        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1665        self.options.append(
1666            optparse.make_option("--destination", dest="cloneDestination",
1667                                 action='store', default=None,
1668                                 help="where to leave result of the clone"))
1669        self.cloneDestination = None
1670        self.needsGit = False
1671
1672    def defaultDestination(self, args):
1673        ## TODO: use common prefix of args?
1674        depotPath = args[0]
1675        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1676        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1677        depotDir = re.sub(r"\.\.\.$,", "", depotDir)
1678        depotDir = re.sub(r"/$", "", depotDir)
1679        return os.path.split(depotDir)[1]
1680
1681    def run(self, args):
1682        if len(args) < 1:
1683            return False
1684
1685        if self.keepRepoPath and not self.cloneDestination:
1686            sys.stderr.write("Must specify destination for --keep-path\n")
1687            sys.exit(1)
1688
1689        depotPaths = args
1690
1691        if not self.cloneDestination and len(depotPaths) > 1:
1692            self.cloneDestination = depotPaths[-1]
1693            depotPaths = depotPaths[:-1]
1694
1695        for p in depotPaths:
1696            if not p.startswith("//"):
1697                return False
1698
1699        if not self.cloneDestination:
1700            self.cloneDestination = self.defaultDestination(args)
1701
1702        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1703        if not os.path.exists(self.cloneDestination):
1704            os.makedirs(self.cloneDestination)
1705        os.chdir(self.cloneDestination)
1706        system("git init")
1707        self.gitdir = os.getcwd() + "/.git"
1708        if not P4Sync.run(self, depotPaths):
1709            return False
1710        if self.branch != "master":
1711            if gitBranchExists("refs/remotes/p4/master"):
1712                system("git branch master refs/remotes/p4/master")
1713                system("git checkout -f")
1714            else:
1715                print "Could not detect main branch. No checkout/master branch created."
1716
1717        return True
1718
1719class P4Branches(Command):
1720    def __init__(self):
1721        Command.__init__(self)
1722        self.options = [ ]
1723        self.description = ("Shows the git branches that hold imports and their "
1724                            + "corresponding perforce depot paths")
1725        self.verbose = False
1726
1727    def run(self, args):
1728        if originP4BranchesExist():
1729            createOrUpdateBranchesFromOrigin()
1730
1731        cmdline = "git rev-parse --symbolic "
1732        cmdline += " --remotes"
1733
1734        for line in read_pipe_lines(cmdline):
1735            line = line.strip()
1736
1737            if not line.startswith('p4/') or line == "p4/HEAD":
1738                continue
1739            branch = line
1740
1741            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1742            settings = extractSettingsGitLog(log)
1743
1744            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1745        return True
1746
1747class HelpFormatter(optparse.IndentedHelpFormatter):
1748    def __init__(self):
1749        optparse.IndentedHelpFormatter.__init__(self)
1750
1751    def format_description(self, description):
1752        if description:
1753            return description + "\n"
1754        else:
1755            return ""
1756
1757def printUsage(commands):
1758    print "usage: %s <command> [options]" % sys.argv[0]
1759    print ""
1760    print "valid commands: %s" % ", ".join(commands)
1761    print ""
1762    print "Try %s <command> --help for command specific help." % sys.argv[0]
1763    print ""
1764
1765commands = {
1766    "debug" : P4Debug,
1767    "submit" : P4Submit,
1768    "commit" : P4Submit,
1769    "sync" : P4Sync,
1770    "rebase" : P4Rebase,
1771    "clone" : P4Clone,
1772    "rollback" : P4RollBack,
1773    "branches" : P4Branches
1774}
1775
1776
1777def main():
1778    if len(sys.argv[1:]) == 0:
1779        printUsage(commands.keys())
1780        sys.exit(2)
1781
1782    cmd = ""
1783    cmdName = sys.argv[1]
1784    try:
1785        klass = commands[cmdName]
1786        cmd = klass()
1787    except KeyError:
1788        print "unknown command %s" % cmdName
1789        print ""
1790        printUsage(commands.keys())
1791        sys.exit(2)
1792
1793    options = cmd.options
1794    cmd.gitdir = os.environ.get("GIT_DIR", None)
1795
1796    args = sys.argv[2:]
1797
1798    if len(options) > 0:
1799        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1800
1801        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1802                                       options,
1803                                       description = cmd.description,
1804                                       formatter = HelpFormatter())
1805
1806        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1807    global verbose
1808    verbose = cmd.verbose
1809    if cmd.needsGit:
1810        if cmd.gitdir == None:
1811            cmd.gitdir = os.path.abspath(".git")
1812            if not isValidGitDir(cmd.gitdir):
1813                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1814                if os.path.exists(cmd.gitdir):
1815                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1816                    if len(cdup) > 0:
1817                        os.chdir(cdup);
1818
1819        if not isValidGitDir(cmd.gitdir):
1820            if isValidGitDir(cmd.gitdir + "/.git"):
1821                cmd.gitdir += "/.git"
1822            else:
1823                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1824
1825        os.environ["GIT_DIR"] = cmd.gitdir
1826
1827    if not cmd.run(args):
1828        parser.print_help()
1829
1830
1831if __name__ == '__main__':
1832    main()