Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'maint' of git://repo.or.cz/git-gui into maint (e98d6df)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19def die(msg):
  20    if verbose:
  21        raise Exception(msg)
  22    else:
  23        sys.stderr.write(msg + "\n")
  24        sys.exit(1)
  25
  26def write_pipe(c, str):
  27    if verbose:
  28        sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30    pipe = os.popen(c, 'w')
  31    val = pipe.write(str)
  32    if pipe.close():
  33        die('Command failed: %s' % c)
  34
  35    return val
  36
  37def read_pipe(c, ignore_error=False):
  38    if verbose:
  39        sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41    pipe = os.popen(c, 'rb')
  42    val = pipe.read()
  43    if pipe.close() and not ignore_error:
  44        die('Command failed: %s' % c)
  45
  46    return val
  47
  48
  49def read_pipe_lines(c):
  50    if verbose:
  51        sys.stderr.write('Reading pipe: %s\n' % c)
  52    ## todo: check return status
  53    pipe = os.popen(c, 'rb')
  54    val = pipe.readlines()
  55    if pipe.close():
  56        die('Command failed: %s' % c)
  57
  58    return val
  59
  60def system(cmd):
  61    if verbose:
  62        sys.stderr.write("executing %s\n" % cmd)
  63    if os.system(cmd) != 0:
  64        die("command failed: %s" % cmd)
  65
  66def isP4Exec(kind):
  67    """Determine if a Perforce 'kind' should have execute permission
  68
  69    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70    or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71    a plus sign, it is also executable"""
  72    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74def setP4ExecBit(file, mode):
  75    # Reopens an already open file and changes the execute bit to match
  76    # the execute bit setting in the passed in mode.
  77
  78    p4Type = "+x"
  79
  80    if not isModeExec(mode):
  81        p4Type = getP4OpenedType(file)
  82        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84        if p4Type[-1] == "+":
  85            p4Type = p4Type[0:-1]
  86
  87    system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89def getP4OpenedType(file):
  90    # Returns the perforce file type for the given file.
  91
  92    result = read_pipe("p4 opened %s" % file)
  93    match = re.match(".*\((.+)\)$", result)
  94    if match:
  95        return match.group(1)
  96    else:
  97        die("Could not determine file type for %s" % file)
  98
  99def diffTreePattern():
 100    # This is a simple generator for the diff tree regex pattern. This could be
 101    # a class variable if this and parseDiffTreeEntry were a part of a class.
 102    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103    while True:
 104        yield pattern
 105
 106def parseDiffTreeEntry(entry):
 107    """Parses a single diff tree entry into its component elements.
 108
 109    See git-diff-tree(1) manpage for details about the format of the diff
 110    output. This method returns a dictionary with the following elements:
 111
 112    src_mode - The mode of the source file
 113    dst_mode - The mode of the destination file
 114    src_sha1 - The sha1 for the source file
 115    dst_sha1 - The sha1 fr the destination file
 116    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117    status_score - The score for the status (applicable for 'C' and 'R'
 118                   statuses). This is None if there is no score.
 119    src - The path for the source file.
 120    dst - The path for the destination file. This is only present for
 121          copy or renames. If it is not present, this is None.
 122
 123    If the pattern is not matched, None is returned."""
 124
 125    match = diffTreePattern().next().match(entry)
 126    if match:
 127        return {
 128            'src_mode': match.group(1),
 129            'dst_mode': match.group(2),
 130            'src_sha1': match.group(3),
 131            'dst_sha1': match.group(4),
 132            'status': match.group(5),
 133            'status_score': match.group(6),
 134            'src': match.group(7),
 135            'dst': match.group(10)
 136        }
 137    return None
 138
 139def isModeExec(mode):
 140    # Returns True if the given git mode represents an executable file,
 141    # otherwise False.
 142    return mode[-3:] == "755"
 143
 144def isModeExecChanged(src_mode, dst_mode):
 145    return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148    cmd = "p4 -G %s" % cmd
 149    if verbose:
 150        sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152    # Use a temporary file to avoid deadlocks without
 153    # subprocess.communicate(), which would put another copy
 154    # of stdout into memory.
 155    stdin_file = None
 156    if stdin is not None:
 157        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158        stdin_file.write(stdin)
 159        stdin_file.flush()
 160        stdin_file.seek(0)
 161
 162    p4 = subprocess.Popen(cmd, shell=True,
 163                          stdin=stdin_file,
 164                          stdout=subprocess.PIPE)
 165
 166    result = []
 167    try:
 168        while True:
 169            entry = marshal.load(p4.stdout)
 170            result.append(entry)
 171    except EOFError:
 172        pass
 173    exitCode = p4.wait()
 174    if exitCode != 0:
 175        entry = {}
 176        entry["p4ExitCode"] = exitCode
 177        result.append(entry)
 178
 179    return result
 180
 181def p4Cmd(cmd):
 182    list = p4CmdList(cmd)
 183    result = {}
 184    for entry in list:
 185        result.update(entry)
 186    return result;
 187
 188def p4Where(depotPath):
 189    if not depotPath.endswith("/"):
 190        depotPath += "/"
 191    output = p4Cmd("where %s..." % depotPath)
 192    if output["code"] == "error":
 193        return ""
 194    clientPath = ""
 195    if "path" in output:
 196        clientPath = output.get("path")
 197    elif "data" in output:
 198        data = output.get("data")
 199        lastSpace = data.rfind(" ")
 200        clientPath = data[lastSpace + 1:]
 201
 202    if clientPath.endswith("..."):
 203        clientPath = clientPath[:-3]
 204    return clientPath
 205
 206def currentGitBranch():
 207    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209def isValidGitDir(path):
 210    if (os.path.exists(path + "/HEAD")
 211        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212        return True;
 213    return False
 214
 215def parseRevision(ref):
 216    return read_pipe("git rev-parse %s" % ref).strip()
 217
 218def extractLogMessageFromGitCommit(commit):
 219    logMessage = ""
 220
 221    ## fixme: title is first line of commit, not 1st paragraph.
 222    foundTitle = False
 223    for log in read_pipe_lines("git cat-file commit %s" % commit):
 224       if not foundTitle:
 225           if len(log) == 1:
 226               foundTitle = True
 227           continue
 228
 229       logMessage += log
 230    return logMessage
 231
 232def extractSettingsGitLog(log):
 233    values = {}
 234    for line in log.split("\n"):
 235        line = line.strip()
 236        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237        if not m:
 238            continue
 239
 240        assignments = m.group(1).split (':')
 241        for a in assignments:
 242            vals = a.split ('=')
 243            key = vals[0].strip()
 244            val = ('='.join (vals[1:])).strip()
 245            if val.endswith ('\"') and val.startswith('"'):
 246                val = val[1:-1]
 247
 248            values[key] = val
 249
 250    paths = values.get("depot-paths")
 251    if not paths:
 252        paths = values.get("depot-path")
 253    if paths:
 254        values['depot-paths'] = paths.split(',')
 255    return values
 256
 257def gitBranchExists(branch):
 258    proc = subprocess.Popen(["git", "rev-parse", branch],
 259                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260    return proc.wait() == 0;
 261
 262def gitConfig(key):
 263    return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265def p4BranchesInGit(branchesAreInRemotes = True):
 266    branches = {}
 267
 268    cmdline = "git rev-parse --symbolic "
 269    if branchesAreInRemotes:
 270        cmdline += " --remotes"
 271    else:
 272        cmdline += " --branches"
 273
 274    for line in read_pipe_lines(cmdline):
 275        line = line.strip()
 276
 277        ## only import to p4/
 278        if not line.startswith('p4/') or line == "p4/HEAD":
 279            continue
 280        branch = line
 281
 282        # strip off p4
 283        branch = re.sub ("^p4/", "", line)
 284
 285        branches[branch] = parseRevision(line)
 286    return branches
 287
 288def findUpstreamBranchPoint(head = "HEAD"):
 289    branches = p4BranchesInGit()
 290    # map from depot-path to branch name
 291    branchByDepotPath = {}
 292    for branch in branches.keys():
 293        tip = branches[branch]
 294        log = extractLogMessageFromGitCommit(tip)
 295        settings = extractSettingsGitLog(log)
 296        if settings.has_key("depot-paths"):
 297            paths = ",".join(settings["depot-paths"])
 298            branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300    settings = None
 301    parent = 0
 302    while parent < 65535:
 303        commit = head + "~%s" % parent
 304        log = extractLogMessageFromGitCommit(commit)
 305        settings = extractSettingsGitLog(log)
 306        if settings.has_key("depot-paths"):
 307            paths = ",".join(settings["depot-paths"])
 308            if branchByDepotPath.has_key(paths):
 309                return [branchByDepotPath[paths], settings]
 310
 311        parent = parent + 1
 312
 313    return ["", settings]
 314
 315def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316    if not silent:
 317        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318               % localRefPrefix)
 319
 320    originPrefix = "origin/p4/"
 321
 322    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323        line = line.strip()
 324        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325            continue
 326
 327        headName = line[len(originPrefix):]
 328        remoteHead = localRefPrefix + headName
 329        originHead = line
 330
 331        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332        if (not original.has_key('depot-paths')
 333            or not original.has_key('change')):
 334            continue
 335
 336        update = False
 337        if not gitBranchExists(remoteHead):
 338            if verbose:
 339                print "creating %s" % remoteHead
 340            update = True
 341        else:
 342            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343            if settings.has_key('change') > 0:
 344                if settings['depot-paths'] == original['depot-paths']:
 345                    originP4Change = int(original['change'])
 346                    p4Change = int(settings['change'])
 347                    if originP4Change > p4Change:
 348                        print ("%s (%s) is newer than %s (%s). "
 349                               "Updating p4 branch from origin."
 350                               % (originHead, originP4Change,
 351                                  remoteHead, p4Change))
 352                        update = True
 353                else:
 354                    print ("Ignoring: %s was imported from %s while "
 355                           "%s was imported from %s"
 356                           % (originHead, ','.join(original['depot-paths']),
 357                              remoteHead, ','.join(settings['depot-paths'])))
 358
 359        if update:
 360            system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362def originP4BranchesExist():
 363        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365def p4ChangesForPaths(depotPaths, changeRange):
 366    assert depotPaths
 367    output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                        for p in depotPaths]))
 369
 370    changes = []
 371    for line in output:
 372        changeNum = line.split(" ")[1]
 373        changes.append(int(changeNum))
 374
 375    changes.sort()
 376    return changes
 377
 378class Command:
 379    def __init__(self):
 380        self.usage = "usage: %prog [options]"
 381        self.needsGit = True
 382
 383class P4Debug(Command):
 384    def __init__(self):
 385        Command.__init__(self)
 386        self.options = [
 387            optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                 default=False),
 389            ]
 390        self.description = "A tool to debug the output of p4 -G."
 391        self.needsGit = False
 392        self.verbose = False
 393
 394    def run(self, args):
 395        j = 0
 396        for output in p4CmdList(" ".join(args)):
 397            print 'Element: %d' % j
 398            j += 1
 399            print output
 400        return True
 401
 402class P4RollBack(Command):
 403    def __init__(self):
 404        Command.__init__(self)
 405        self.options = [
 406            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408        ]
 409        self.description = "A tool to debug the multi-branch import. Don't use :)"
 410        self.verbose = False
 411        self.rollbackLocalBranches = False
 412
 413    def run(self, args):
 414        if len(args) != 1:
 415            return False
 416        maxChange = int(args[0])
 417
 418        if "p4ExitCode" in p4Cmd("changes -m 1"):
 419            die("Problems executing p4");
 420
 421        if self.rollbackLocalBranches:
 422            refPrefix = "refs/heads/"
 423            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424        else:
 425            refPrefix = "refs/remotes/"
 426            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428        for line in lines:
 429            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                line = line.strip()
 431                ref = refPrefix + line
 432                log = extractLogMessageFromGitCommit(ref)
 433                settings = extractSettingsGitLog(log)
 434
 435                depotPaths = settings['depot-paths']
 436                change = settings['change']
 437
 438                changed = False
 439
 440                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                           for p in depotPaths]))) == 0:
 442                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                    continue
 445
 446                while change and int(change) > maxChange:
 447                    changed = True
 448                    if self.verbose:
 449                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                    system("git update-ref %s \"%s^\"" % (ref, ref))
 451                    log = extractLogMessageFromGitCommit(ref)
 452                    settings =  extractSettingsGitLog(log)
 453
 454
 455                    depotPaths = settings['depot-paths']
 456                    change = settings['change']
 457
 458                if changed:
 459                    print "%s rewound to %s" % (ref, change)
 460
 461        return True
 462
 463class P4Submit(Command):
 464    def __init__(self):
 465        Command.__init__(self)
 466        self.options = [
 467                optparse.make_option("--continue", action="store_false", dest="firstTime"),
 468                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 469                optparse.make_option("--origin", dest="origin"),
 470                optparse.make_option("--reset", action="store_true", dest="reset"),
 471                optparse.make_option("--log-substitutions", dest="substFile"),
 472                optparse.make_option("--dry-run", action="store_true"),
 473                optparse.make_option("--direct", dest="directSubmit", action="store_true"),
 474                optparse.make_option("--trust-me-like-a-fool", dest="trustMeLikeAFool", action="store_true"),
 475                optparse.make_option("-M", dest="detectRename", action="store_true"),
 476        ]
 477        self.description = "Submit changes from git to the perforce depot."
 478        self.usage += " [name of git branch to submit into perforce depot]"
 479        self.firstTime = True
 480        self.reset = False
 481        self.interactive = True
 482        self.dryRun = False
 483        self.substFile = ""
 484        self.firstTime = True
 485        self.origin = ""
 486        self.directSubmit = False
 487        self.trustMeLikeAFool = False
 488        self.detectRename = False
 489        self.verbose = False
 490        self.isWindows = (platform.system() == "Windows")
 491
 492        self.logSubstitutions = {}
 493        self.logSubstitutions["<enter description here>"] = "%log%"
 494        self.logSubstitutions["\tDetails:"] = "\tDetails:  %log%"
 495
 496    def check(self):
 497        if len(p4CmdList("opened ...")) > 0:
 498            die("You have files opened with perforce! Close them before starting the sync.")
 499
 500    def start(self):
 501        if len(self.config) > 0 and not self.reset:
 502            die("Cannot start sync. Previous sync config found at %s\n"
 503                "If you want to start submitting again from scratch "
 504                "maybe you want to call git-p4 submit --reset" % self.configFile)
 505
 506        commits = []
 507        if self.directSubmit:
 508            commits.append("0")
 509        else:
 510            for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 511                commits.append(line.strip())
 512            commits.reverse()
 513
 514        self.config["commits"] = commits
 515
 516    def prepareLogMessage(self, template, message):
 517        result = ""
 518
 519        for line in template.split("\n"):
 520            if line.startswith("#"):
 521                result += line + "\n"
 522                continue
 523
 524            substituted = False
 525            for key in self.logSubstitutions.keys():
 526                if line.find(key) != -1:
 527                    value = self.logSubstitutions[key]
 528                    value = value.replace("%log%", message)
 529                    if value != "@remove@":
 530                        result += line.replace(key, value) + "\n"
 531                    substituted = True
 532                    break
 533
 534            if not substituted:
 535                result += line + "\n"
 536
 537        return result
 538
 539    def prepareSubmitTemplate(self):
 540        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 541        template = ""
 542        inFilesSection = False
 543        for line in read_pipe_lines("p4 change -o"):
 544            if inFilesSection:
 545                if line.startswith("\t"):
 546                    # path starts and ends with a tab
 547                    path = line[1:]
 548                    lastTab = path.rfind("\t")
 549                    if lastTab != -1:
 550                        path = path[:lastTab]
 551                        if not path.startswith(self.depotPath):
 552                            continue
 553                else:
 554                    inFilesSection = False
 555            else:
 556                if line.startswith("Files:"):
 557                    inFilesSection = True
 558
 559            template += line
 560
 561        return template
 562
 563    def applyCommit(self, id):
 564        if self.directSubmit:
 565            print "Applying local change in working directory/index"
 566            diff = self.diffStatus
 567        else:
 568            print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 569            diffOpts = ("", "-M")[self.detectRename]
 570            diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 571        filesToAdd = set()
 572        filesToDelete = set()
 573        editedFiles = set()
 574        filesToChangeExecBit = {}
 575        for line in diff:
 576            diff = parseDiffTreeEntry(line)
 577            modifier = diff['status']
 578            path = diff['src']
 579            if modifier == "M":
 580                system("p4 edit \"%s\"" % path)
 581                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 582                    filesToChangeExecBit[path] = diff['dst_mode']
 583                editedFiles.add(path)
 584            elif modifier == "A":
 585                filesToAdd.add(path)
 586                filesToChangeExecBit[path] = diff['dst_mode']
 587                if path in filesToDelete:
 588                    filesToDelete.remove(path)
 589            elif modifier == "D":
 590                filesToDelete.add(path)
 591                if path in filesToAdd:
 592                    filesToAdd.remove(path)
 593            elif modifier == "R":
 594                src, dest = diff['src'], diff['dst']
 595                system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 596                system("p4 edit \"%s\"" % (dest))
 597                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 598                    filesToChangeExecBit[dest] = diff['dst_mode']
 599                os.unlink(dest)
 600                editedFiles.add(dest)
 601                filesToDelete.add(src)
 602            else:
 603                die("unknown modifier %s for %s" % (modifier, path))
 604
 605        if self.directSubmit:
 606            diffcmd = "cat \"%s\"" % self.diffFile
 607        else:
 608            diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 609        patchcmd = diffcmd + " | git apply "
 610        tryPatchCmd = patchcmd + "--check -"
 611        applyPatchCmd = patchcmd + "--check --apply -"
 612
 613        if os.system(tryPatchCmd) != 0:
 614            print "Unfortunately applying the change failed!"
 615            print "What do you want to do?"
 616            response = "x"
 617            while response != "s" and response != "a" and response != "w":
 618                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 619                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 620            if response == "s":
 621                print "Skipping! Good luck with the next patches..."
 622                for f in editedFiles:
 623                    system("p4 revert \"%s\"" % f);
 624                for f in filesToAdd:
 625                    system("rm %s" %f)
 626                return
 627            elif response == "a":
 628                os.system(applyPatchCmd)
 629                if len(filesToAdd) > 0:
 630                    print "You may also want to call p4 add on the following files:"
 631                    print " ".join(filesToAdd)
 632                if len(filesToDelete):
 633                    print "The following files should be scheduled for deletion with p4 delete:"
 634                    print " ".join(filesToDelete)
 635                die("Please resolve and submit the conflict manually and "
 636                    + "continue afterwards with git-p4 submit --continue")
 637            elif response == "w":
 638                system(diffcmd + " > patch.txt")
 639                print "Patch saved to patch.txt in %s !" % self.clientPath
 640                die("Please resolve and submit the conflict manually and "
 641                    "continue afterwards with git-p4 submit --continue")
 642
 643        system(applyPatchCmd)
 644
 645        for f in filesToAdd:
 646            system("p4 add \"%s\"" % f)
 647        for f in filesToDelete:
 648            system("p4 revert \"%s\"" % f)
 649            system("p4 delete \"%s\"" % f)
 650
 651        # Set/clear executable bits
 652        for f in filesToChangeExecBit.keys():
 653            mode = filesToChangeExecBit[f]
 654            setP4ExecBit(f, mode)
 655
 656        logMessage = ""
 657        if not self.directSubmit:
 658            logMessage = extractLogMessageFromGitCommit(id)
 659            logMessage = logMessage.replace("\n", "\n\t")
 660            if self.isWindows:
 661                logMessage = logMessage.replace("\n", "\r\n")
 662            logMessage = logMessage.strip()
 663
 664        template = self.prepareSubmitTemplate()
 665
 666        if self.interactive:
 667            submitTemplate = self.prepareLogMessage(template, logMessage)
 668            diff = read_pipe("p4 diff -du ...")
 669
 670            for newFile in filesToAdd:
 671                diff += "==== new file ====\n"
 672                diff += "--- /dev/null\n"
 673                diff += "+++ %s\n" % newFile
 674                f = open(newFile, "r")
 675                for line in f.readlines():
 676                    diff += "+" + line
 677                f.close()
 678
 679            separatorLine = "######## everything below this line is just the diff #######"
 680            if platform.system() == "Windows":
 681                separatorLine += "\r"
 682            separatorLine += "\n"
 683
 684            response = "e"
 685            if self.trustMeLikeAFool:
 686                response = "y"
 687
 688            firstIteration = True
 689            while response == "e":
 690                if not firstIteration:
 691                    response = raw_input("Do you want to submit this change? [y]es/[e]dit/[n]o/[s]kip ")
 692                firstIteration = False
 693                if response == "e":
 694                    [handle, fileName] = tempfile.mkstemp()
 695                    tmpFile = os.fdopen(handle, "w+")
 696                    tmpFile.write(submitTemplate + separatorLine + diff)
 697                    tmpFile.close()
 698                    defaultEditor = "vi"
 699                    if platform.system() == "Windows":
 700                        defaultEditor = "notepad"
 701                    editor = os.environ.get("EDITOR", defaultEditor);
 702                    system(editor + " " + fileName)
 703                    tmpFile = open(fileName, "rb")
 704                    message = tmpFile.read()
 705                    tmpFile.close()
 706                    os.remove(fileName)
 707                    submitTemplate = message[:message.index(separatorLine)]
 708                    if self.isWindows:
 709                        submitTemplate = submitTemplate.replace("\r\n", "\n")
 710
 711            if response == "y" or response == "yes":
 712               if self.dryRun:
 713                   print submitTemplate
 714                   raw_input("Press return to continue...")
 715               else:
 716                   if self.directSubmit:
 717                       print "Submitting to git first"
 718                       os.chdir(self.oldWorkingDirectory)
 719                       write_pipe("git commit -a -F -", submitTemplate)
 720                       os.chdir(self.clientPath)
 721
 722                   write_pipe("p4 submit -i", submitTemplate)
 723            elif response == "s":
 724                for f in editedFiles:
 725                    system("p4 revert \"%s\"" % f);
 726                for f in filesToAdd:
 727                    system("p4 revert \"%s\"" % f);
 728                    system("rm %s" %f)
 729                for f in filesToDelete:
 730                    system("p4 delete \"%s\"" % f);
 731                return
 732            else:
 733                print "Not submitting!"
 734                self.interactive = False
 735        else:
 736            fileName = "submit.txt"
 737            file = open(fileName, "w+")
 738            file.write(self.prepareLogMessage(template, logMessage))
 739            file.close()
 740            print ("Perforce submit template written as %s. "
 741                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 742                   % (fileName, fileName))
 743
 744    def run(self, args):
 745        if len(args) == 0:
 746            self.master = currentGitBranch()
 747            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 748                die("Detecting current git branch failed!")
 749        elif len(args) == 1:
 750            self.master = args[0]
 751        else:
 752            return False
 753
 754        [upstream, settings] = findUpstreamBranchPoint()
 755        self.depotPath = settings['depot-paths'][0]
 756        if len(self.origin) == 0:
 757            self.origin = upstream
 758
 759        if self.verbose:
 760            print "Origin branch is " + self.origin
 761
 762        if len(self.depotPath) == 0:
 763            print "Internal error: cannot locate perforce depot path from existing branches"
 764            sys.exit(128)
 765
 766        self.clientPath = p4Where(self.depotPath)
 767
 768        if len(self.clientPath) == 0:
 769            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 770            sys.exit(128)
 771
 772        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 773        self.oldWorkingDirectory = os.getcwd()
 774
 775        if self.directSubmit:
 776            self.diffStatus = read_pipe_lines("git diff -r --name-status HEAD")
 777            if len(self.diffStatus) == 0:
 778                print "No changes in working directory to submit."
 779                return True
 780            patch = read_pipe("git diff -p --binary --diff-filter=ACMRTUXB HEAD")
 781            self.diffFile = self.gitdir + "/p4-git-diff"
 782            f = open(self.diffFile, "wb")
 783            f.write(patch)
 784            f.close();
 785
 786        os.chdir(self.clientPath)
 787        print "Syncronizing p4 checkout..."
 788        system("p4 sync ...")
 789
 790        if self.reset:
 791            self.firstTime = True
 792
 793        if len(self.substFile) > 0:
 794            for line in open(self.substFile, "r").readlines():
 795                tokens = line.strip().split("=")
 796                self.logSubstitutions[tokens[0]] = tokens[1]
 797
 798        self.check()
 799        self.configFile = self.gitdir + "/p4-git-sync.cfg"
 800        self.config = shelve.open(self.configFile, writeback=True)
 801
 802        if self.firstTime:
 803            self.start()
 804
 805        commits = self.config.get("commits", [])
 806
 807        while len(commits) > 0:
 808            self.firstTime = False
 809            commit = commits[0]
 810            commits = commits[1:]
 811            self.config["commits"] = commits
 812            self.applyCommit(commit)
 813            if not self.interactive:
 814                break
 815
 816        self.config.close()
 817
 818        if self.directSubmit:
 819            os.remove(self.diffFile)
 820
 821        if len(commits) == 0:
 822            if self.firstTime:
 823                print "No changes found to apply between %s and current HEAD" % self.origin
 824            else:
 825                print "All changes applied!"
 826                os.chdir(self.oldWorkingDirectory)
 827
 828                sync = P4Sync()
 829                sync.run([])
 830
 831                response = raw_input("Do you want to rebase current HEAD from Perforce now using git-p4 rebase? [y]es/[n]o ")
 832                if response == "y" or response == "yes":
 833                    rebase = P4Rebase()
 834                    rebase.rebase()
 835            os.remove(self.configFile)
 836
 837        return True
 838
 839class P4Sync(Command):
 840    def __init__(self):
 841        Command.__init__(self)
 842        self.options = [
 843                optparse.make_option("--branch", dest="branch"),
 844                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 845                optparse.make_option("--changesfile", dest="changesFile"),
 846                optparse.make_option("--silent", dest="silent", action="store_true"),
 847                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 848                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 849                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 850                                     help="Import into refs/heads/ , not refs/remotes"),
 851                optparse.make_option("--max-changes", dest="maxChanges"),
 852                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 853                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import")
 854        ]
 855        self.description = """Imports from Perforce into a git repository.\n
 856    example:
 857    //depot/my/project/ -- to import the current head
 858    //depot/my/project/@all -- to import everything
 859    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 860
 861    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 862
 863        self.usage += " //depot/path[@revRange]"
 864        self.silent = False
 865        self.createdBranches = Set()
 866        self.committedChanges = Set()
 867        self.branch = ""
 868        self.detectBranches = False
 869        self.detectLabels = False
 870        self.changesFile = ""
 871        self.syncWithOrigin = True
 872        self.verbose = False
 873        self.importIntoRemotes = True
 874        self.maxChanges = ""
 875        self.isWindows = (platform.system() == "Windows")
 876        self.keepRepoPath = False
 877        self.depotPaths = None
 878        self.p4BranchesInGit = []
 879
 880        if gitConfig("git-p4.syncFromOrigin") == "false":
 881            self.syncWithOrigin = False
 882
 883    def extractFilesFromCommit(self, commit):
 884        files = []
 885        fnum = 0
 886        while commit.has_key("depotFile%s" % fnum):
 887            path =  commit["depotFile%s" % fnum]
 888
 889            found = [p for p in self.depotPaths
 890                     if path.startswith (p)]
 891            if not found:
 892                fnum = fnum + 1
 893                continue
 894
 895            file = {}
 896            file["path"] = path
 897            file["rev"] = commit["rev%s" % fnum]
 898            file["action"] = commit["action%s" % fnum]
 899            file["type"] = commit["type%s" % fnum]
 900            files.append(file)
 901            fnum = fnum + 1
 902        return files
 903
 904    def stripRepoPath(self, path, prefixes):
 905        if self.keepRepoPath:
 906            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 907
 908        for p in prefixes:
 909            if path.startswith(p):
 910                path = path[len(p):]
 911
 912        return path
 913
 914    def splitFilesIntoBranches(self, commit):
 915        branches = {}
 916        fnum = 0
 917        while commit.has_key("depotFile%s" % fnum):
 918            path =  commit["depotFile%s" % fnum]
 919            found = [p for p in self.depotPaths
 920                     if path.startswith (p)]
 921            if not found:
 922                fnum = fnum + 1
 923                continue
 924
 925            file = {}
 926            file["path"] = path
 927            file["rev"] = commit["rev%s" % fnum]
 928            file["action"] = commit["action%s" % fnum]
 929            file["type"] = commit["type%s" % fnum]
 930            fnum = fnum + 1
 931
 932            relPath = self.stripRepoPath(path, self.depotPaths)
 933
 934            for branch in self.knownBranches.keys():
 935
 936                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 937                if relPath.startswith(branch + "/"):
 938                    if branch not in branches:
 939                        branches[branch] = []
 940                    branches[branch].append(file)
 941                    break
 942
 943        return branches
 944
 945    ## Should move this out, doesn't use SELF.
 946    def readP4Files(self, files):
 947        files = [f for f in files
 948                 if f['action'] != 'delete']
 949
 950        if not files:
 951            return
 952
 953        filedata = p4CmdList('-x - print',
 954                             stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 955                                              for f in files]),
 956                             stdin_mode='w+')
 957        if "p4ExitCode" in filedata[0]:
 958            die("Problems executing p4. Error: [%d]."
 959                % (filedata[0]['p4ExitCode']));
 960
 961        j = 0;
 962        contents = {}
 963        while j < len(filedata):
 964            stat = filedata[j]
 965            j += 1
 966            text = ''
 967            while j < len(filedata) and filedata[j]['code'] in ('text',
 968                                                                'binary'):
 969                text += filedata[j]['data']
 970                j += 1
 971
 972
 973            if not stat.has_key('depotFile'):
 974                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 975                continue
 976
 977            contents[stat['depotFile']] = text
 978
 979        for f in files:
 980            assert not f.has_key('data')
 981            f['data'] = contents[f['path']]
 982
 983    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 984        epoch = details["time"]
 985        author = details["user"]
 986
 987        if self.verbose:
 988            print "commit into %s" % branch
 989
 990        # start with reading files; if that fails, we should not
 991        # create a commit.
 992        new_files = []
 993        for f in files:
 994            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 995                new_files.append (f)
 996            else:
 997                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 998        files = new_files
 999        self.readP4Files(files)
1000
1001
1002
1003
1004        self.gitStream.write("commit %s\n" % branch)
1005#        gitStream.write("mark :%s\n" % details["change"])
1006        self.committedChanges.add(int(details["change"]))
1007        committer = ""
1008        if author not in self.users:
1009            self.getUserMapFromPerforceServer()
1010        if author in self.users:
1011            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1012        else:
1013            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1014
1015        self.gitStream.write("committer %s\n" % committer)
1016
1017        self.gitStream.write("data <<EOT\n")
1018        self.gitStream.write(details["desc"])
1019        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1020                             % (','.join (branchPrefixes), details["change"]))
1021        if len(details['options']) > 0:
1022            self.gitStream.write(": options = %s" % details['options'])
1023        self.gitStream.write("]\nEOT\n\n")
1024
1025        if len(parent) > 0:
1026            if self.verbose:
1027                print "parent %s" % parent
1028            self.gitStream.write("from %s\n" % parent)
1029
1030        for file in files:
1031            if file["type"] == "apple":
1032                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1033                continue
1034
1035            relPath = self.stripRepoPath(file['path'], branchPrefixes)
1036            if file["action"] == "delete":
1037                self.gitStream.write("D %s\n" % relPath)
1038            else:
1039                data = file['data']
1040
1041                mode = "644"
1042                if isP4Exec(file["type"]):
1043                    mode = "755"
1044                elif file["type"] == "symlink":
1045                    mode = "120000"
1046                    # p4 print on a symlink contains "target\n", so strip it off
1047                    data = data[:-1]
1048
1049                if self.isWindows and file["type"].endswith("text"):
1050                    data = data.replace("\r\n", "\n")
1051
1052                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1053                self.gitStream.write("data %s\n" % len(data))
1054                self.gitStream.write(data)
1055                self.gitStream.write("\n")
1056
1057        self.gitStream.write("\n")
1058
1059        change = int(details["change"])
1060
1061        if self.labels.has_key(change):
1062            label = self.labels[change]
1063            labelDetails = label[0]
1064            labelRevisions = label[1]
1065            if self.verbose:
1066                print "Change %s is labelled %s" % (change, labelDetails)
1067
1068            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1069                                                    for p in branchPrefixes]))
1070
1071            if len(files) == len(labelRevisions):
1072
1073                cleanedFiles = {}
1074                for info in files:
1075                    if info["action"] == "delete":
1076                        continue
1077                    cleanedFiles[info["depotFile"]] = info["rev"]
1078
1079                if cleanedFiles == labelRevisions:
1080                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1081                    self.gitStream.write("from %s\n" % branch)
1082
1083                    owner = labelDetails["Owner"]
1084                    tagger = ""
1085                    if author in self.users:
1086                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1087                    else:
1088                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1089                    self.gitStream.write("tagger %s\n" % tagger)
1090                    self.gitStream.write("data <<EOT\n")
1091                    self.gitStream.write(labelDetails["Description"])
1092                    self.gitStream.write("EOT\n\n")
1093
1094                else:
1095                    if not self.silent:
1096                        print ("Tag %s does not match with change %s: files do not match."
1097                               % (labelDetails["label"], change))
1098
1099            else:
1100                if not self.silent:
1101                    print ("Tag %s does not match with change %s: file count is different."
1102                           % (labelDetails["label"], change))
1103
1104    def getUserCacheFilename(self):
1105        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1106        return home + "/.gitp4-usercache.txt"
1107
1108    def getUserMapFromPerforceServer(self):
1109        if self.userMapFromPerforceServer:
1110            return
1111        self.users = {}
1112
1113        for output in p4CmdList("users"):
1114            if not output.has_key("User"):
1115                continue
1116            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1117
1118
1119        s = ''
1120        for (key, val) in self.users.items():
1121            s += "%s\t%s\n" % (key, val)
1122
1123        open(self.getUserCacheFilename(), "wb").write(s)
1124        self.userMapFromPerforceServer = True
1125
1126    def loadUserMapFromCache(self):
1127        self.users = {}
1128        self.userMapFromPerforceServer = False
1129        try:
1130            cache = open(self.getUserCacheFilename(), "rb")
1131            lines = cache.readlines()
1132            cache.close()
1133            for line in lines:
1134                entry = line.strip().split("\t")
1135                self.users[entry[0]] = entry[1]
1136        except IOError:
1137            self.getUserMapFromPerforceServer()
1138
1139    def getLabels(self):
1140        self.labels = {}
1141
1142        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1143        if len(l) > 0 and not self.silent:
1144            print "Finding files belonging to labels in %s" % `self.depotPaths`
1145
1146        for output in l:
1147            label = output["label"]
1148            revisions = {}
1149            newestChange = 0
1150            if self.verbose:
1151                print "Querying files for label %s" % label
1152            for file in p4CmdList("files "
1153                                  +  ' '.join (["%s...@%s" % (p, label)
1154                                                for p in self.depotPaths])):
1155                revisions[file["depotFile"]] = file["rev"]
1156                change = int(file["change"])
1157                if change > newestChange:
1158                    newestChange = change
1159
1160            self.labels[newestChange] = [output, revisions]
1161
1162        if self.verbose:
1163            print "Label changes: %s" % self.labels.keys()
1164
1165    def guessProjectName(self):
1166        for p in self.depotPaths:
1167            if p.endswith("/"):
1168                p = p[:-1]
1169            p = p[p.strip().rfind("/") + 1:]
1170            if not p.endswith("/"):
1171               p += "/"
1172            return p
1173
1174    def getBranchMapping(self):
1175        lostAndFoundBranches = set()
1176
1177        for info in p4CmdList("branches"):
1178            details = p4Cmd("branch -o %s" % info["branch"])
1179            viewIdx = 0
1180            while details.has_key("View%s" % viewIdx):
1181                paths = details["View%s" % viewIdx].split(" ")
1182                viewIdx = viewIdx + 1
1183                # require standard //depot/foo/... //depot/bar/... mapping
1184                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1185                    continue
1186                source = paths[0]
1187                destination = paths[1]
1188                ## HACK
1189                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1190                    source = source[len(self.depotPaths[0]):-4]
1191                    destination = destination[len(self.depotPaths[0]):-4]
1192
1193                    if destination in self.knownBranches:
1194                        if not self.silent:
1195                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1196                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1197                        continue
1198
1199                    self.knownBranches[destination] = source
1200
1201                    lostAndFoundBranches.discard(destination)
1202
1203                    if source not in self.knownBranches:
1204                        lostAndFoundBranches.add(source)
1205
1206
1207        for branch in lostAndFoundBranches:
1208            self.knownBranches[branch] = branch
1209
1210    def getBranchMappingFromGitBranches(self):
1211        branches = p4BranchesInGit(self.importIntoRemotes)
1212        for branch in branches.keys():
1213            if branch == "master":
1214                branch = "main"
1215            else:
1216                branch = branch[len(self.projectName):]
1217            self.knownBranches[branch] = branch
1218
1219    def listExistingP4GitBranches(self):
1220        # branches holds mapping from name to commit
1221        branches = p4BranchesInGit(self.importIntoRemotes)
1222        self.p4BranchesInGit = branches.keys()
1223        for branch in branches.keys():
1224            self.initialParents[self.refPrefix + branch] = branches[branch]
1225
1226    def updateOptionDict(self, d):
1227        option_keys = {}
1228        if self.keepRepoPath:
1229            option_keys['keepRepoPath'] = 1
1230
1231        d["options"] = ' '.join(sorted(option_keys.keys()))
1232
1233    def readOptions(self, d):
1234        self.keepRepoPath = (d.has_key('options')
1235                             and ('keepRepoPath' in d['options']))
1236
1237    def gitRefForBranch(self, branch):
1238        if branch == "main":
1239            return self.refPrefix + "master"
1240
1241        if len(branch) <= 0:
1242            return branch
1243
1244        return self.refPrefix + self.projectName + branch
1245
1246    def gitCommitByP4Change(self, ref, change):
1247        if self.verbose:
1248            print "looking in ref " + ref + " for change %s using bisect..." % change
1249
1250        earliestCommit = ""
1251        latestCommit = parseRevision(ref)
1252
1253        while True:
1254            if self.verbose:
1255                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1256            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1257            if len(next) == 0:
1258                if self.verbose:
1259                    print "argh"
1260                return ""
1261            log = extractLogMessageFromGitCommit(next)
1262            settings = extractSettingsGitLog(log)
1263            currentChange = int(settings['change'])
1264            if self.verbose:
1265                print "current change %s" % currentChange
1266
1267            if currentChange == change:
1268                if self.verbose:
1269                    print "found %s" % next
1270                return next
1271
1272            if currentChange < change:
1273                earliestCommit = "^%s" % next
1274            else:
1275                latestCommit = "%s" % next
1276
1277        return ""
1278
1279    def importNewBranch(self, branch, maxChange):
1280        # make fast-import flush all changes to disk and update the refs using the checkpoint
1281        # command so that we can try to find the branch parent in the git history
1282        self.gitStream.write("checkpoint\n\n");
1283        self.gitStream.flush();
1284        branchPrefix = self.depotPaths[0] + branch + "/"
1285        range = "@1,%s" % maxChange
1286        #print "prefix" + branchPrefix
1287        changes = p4ChangesForPaths([branchPrefix], range)
1288        if len(changes) <= 0:
1289            return False
1290        firstChange = changes[0]
1291        #print "first change in branch: %s" % firstChange
1292        sourceBranch = self.knownBranches[branch]
1293        sourceDepotPath = self.depotPaths[0] + sourceBranch
1294        sourceRef = self.gitRefForBranch(sourceBranch)
1295        #print "source " + sourceBranch
1296
1297        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1298        #print "branch parent: %s" % branchParentChange
1299        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1300        if len(gitParent) > 0:
1301            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1302            #print "parent git commit: %s" % gitParent
1303
1304        self.importChanges(changes)
1305        return True
1306
1307    def importChanges(self, changes):
1308        cnt = 1
1309        for change in changes:
1310            description = p4Cmd("describe %s" % change)
1311            self.updateOptionDict(description)
1312
1313            if not self.silent:
1314                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1315                sys.stdout.flush()
1316            cnt = cnt + 1
1317
1318            try:
1319                if self.detectBranches:
1320                    branches = self.splitFilesIntoBranches(description)
1321                    for branch in branches.keys():
1322                        ## HACK  --hwn
1323                        branchPrefix = self.depotPaths[0] + branch + "/"
1324
1325                        parent = ""
1326
1327                        filesForCommit = branches[branch]
1328
1329                        if self.verbose:
1330                            print "branch is %s" % branch
1331
1332                        self.updatedBranches.add(branch)
1333
1334                        if branch not in self.createdBranches:
1335                            self.createdBranches.add(branch)
1336                            parent = self.knownBranches[branch]
1337                            if parent == branch:
1338                                parent = ""
1339                            else:
1340                                fullBranch = self.projectName + branch
1341                                if fullBranch not in self.p4BranchesInGit:
1342                                    if not self.silent:
1343                                        print("\n    Importing new branch %s" % fullBranch);
1344                                    if self.importNewBranch(branch, change - 1):
1345                                        parent = ""
1346                                        self.p4BranchesInGit.append(fullBranch)
1347                                    if not self.silent:
1348                                        print("\n    Resuming with change %s" % change);
1349
1350                                if self.verbose:
1351                                    print "parent determined through known branches: %s" % parent
1352
1353                        branch = self.gitRefForBranch(branch)
1354                        parent = self.gitRefForBranch(parent)
1355
1356                        if self.verbose:
1357                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1358
1359                        if len(parent) == 0 and branch in self.initialParents:
1360                            parent = self.initialParents[branch]
1361                            del self.initialParents[branch]
1362
1363                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1364                else:
1365                    files = self.extractFilesFromCommit(description)
1366                    self.commit(description, files, self.branch, self.depotPaths,
1367                                self.initialParent)
1368                    self.initialParent = ""
1369            except IOError:
1370                print self.gitError.read()
1371                sys.exit(1)
1372
1373    def importHeadRevision(self, revision):
1374        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1375
1376        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1377        details["desc"] = ("Initial import of %s from the state at revision %s"
1378                           % (' '.join(self.depotPaths), revision))
1379        details["change"] = revision
1380        newestRevision = 0
1381
1382        fileCnt = 0
1383        for info in p4CmdList("files "
1384                              +  ' '.join(["%s...%s"
1385                                           % (p, revision)
1386                                           for p in self.depotPaths])):
1387
1388            if info['code'] == 'error':
1389                sys.stderr.write("p4 returned an error: %s\n"
1390                                 % info['data'])
1391                sys.exit(1)
1392
1393
1394            change = int(info["change"])
1395            if change > newestRevision:
1396                newestRevision = change
1397
1398            if info["action"] == "delete":
1399                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1400                #fileCnt = fileCnt + 1
1401                continue
1402
1403            for prop in ["depotFile", "rev", "action", "type" ]:
1404                details["%s%s" % (prop, fileCnt)] = info[prop]
1405
1406            fileCnt = fileCnt + 1
1407
1408        details["change"] = newestRevision
1409        self.updateOptionDict(details)
1410        try:
1411            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1412        except IOError:
1413            print "IO error with git fast-import. Is your git version recent enough?"
1414            print self.gitError.read()
1415
1416
1417    def run(self, args):
1418        self.depotPaths = []
1419        self.changeRange = ""
1420        self.initialParent = ""
1421        self.previousDepotPaths = []
1422
1423        # map from branch depot path to parent branch
1424        self.knownBranches = {}
1425        self.initialParents = {}
1426        self.hasOrigin = originP4BranchesExist()
1427        if not self.syncWithOrigin:
1428            self.hasOrigin = False
1429
1430        if self.importIntoRemotes:
1431            self.refPrefix = "refs/remotes/p4/"
1432        else:
1433            self.refPrefix = "refs/heads/p4/"
1434
1435        if self.syncWithOrigin and self.hasOrigin:
1436            if not self.silent:
1437                print "Syncing with origin first by calling git fetch origin"
1438            system("git fetch origin")
1439
1440        if len(self.branch) == 0:
1441            self.branch = self.refPrefix + "master"
1442            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1443                system("git update-ref %s refs/heads/p4" % self.branch)
1444                system("git branch -D p4");
1445            # create it /after/ importing, when master exists
1446            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1447                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1448
1449        # TODO: should always look at previous commits,
1450        # merge with previous imports, if possible.
1451        if args == []:
1452            if self.hasOrigin:
1453                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1454            self.listExistingP4GitBranches()
1455
1456            if len(self.p4BranchesInGit) > 1:
1457                if not self.silent:
1458                    print "Importing from/into multiple branches"
1459                self.detectBranches = True
1460
1461            if self.verbose:
1462                print "branches: %s" % self.p4BranchesInGit
1463
1464            p4Change = 0
1465            for branch in self.p4BranchesInGit:
1466                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1467
1468                settings = extractSettingsGitLog(logMsg)
1469
1470                self.readOptions(settings)
1471                if (settings.has_key('depot-paths')
1472                    and settings.has_key ('change')):
1473                    change = int(settings['change']) + 1
1474                    p4Change = max(p4Change, change)
1475
1476                    depotPaths = sorted(settings['depot-paths'])
1477                    if self.previousDepotPaths == []:
1478                        self.previousDepotPaths = depotPaths
1479                    else:
1480                        paths = []
1481                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1482                            for i in range(0, min(len(cur), len(prev))):
1483                                if cur[i] <> prev[i]:
1484                                    i = i - 1
1485                                    break
1486
1487                            paths.append (cur[:i + 1])
1488
1489                        self.previousDepotPaths = paths
1490
1491            if p4Change > 0:
1492                self.depotPaths = sorted(self.previousDepotPaths)
1493                self.changeRange = "@%s,#head" % p4Change
1494                if not self.detectBranches:
1495                    self.initialParent = parseRevision(self.branch)
1496                if not self.silent and not self.detectBranches:
1497                    print "Performing incremental import into %s git branch" % self.branch
1498
1499        if not self.branch.startswith("refs/"):
1500            self.branch = "refs/heads/" + self.branch
1501
1502        if len(args) == 0 and self.depotPaths:
1503            if not self.silent:
1504                print "Depot paths: %s" % ' '.join(self.depotPaths)
1505        else:
1506            if self.depotPaths and self.depotPaths != args:
1507                print ("previous import used depot path %s and now %s was specified. "
1508                       "This doesn't work!" % (' '.join (self.depotPaths),
1509                                               ' '.join (args)))
1510                sys.exit(1)
1511
1512            self.depotPaths = sorted(args)
1513
1514        revision = ""
1515        self.users = {}
1516
1517        newPaths = []
1518        for p in self.depotPaths:
1519            if p.find("@") != -1:
1520                atIdx = p.index("@")
1521                self.changeRange = p[atIdx:]
1522                if self.changeRange == "@all":
1523                    self.changeRange = ""
1524                elif ',' not in self.changeRange:
1525                    revision = self.changeRange
1526                    self.changeRange = ""
1527                p = p[:atIdx]
1528            elif p.find("#") != -1:
1529                hashIdx = p.index("#")
1530                revision = p[hashIdx:]
1531                p = p[:hashIdx]
1532            elif self.previousDepotPaths == []:
1533                revision = "#head"
1534
1535            p = re.sub ("\.\.\.$", "", p)
1536            if not p.endswith("/"):
1537                p += "/"
1538
1539            newPaths.append(p)
1540
1541        self.depotPaths = newPaths
1542
1543
1544        self.loadUserMapFromCache()
1545        self.labels = {}
1546        if self.detectLabels:
1547            self.getLabels();
1548
1549        if self.detectBranches:
1550            ## FIXME - what's a P4 projectName ?
1551            self.projectName = self.guessProjectName()
1552
1553            if self.hasOrigin:
1554                self.getBranchMappingFromGitBranches()
1555            else:
1556                self.getBranchMapping()
1557            if self.verbose:
1558                print "p4-git branches: %s" % self.p4BranchesInGit
1559                print "initial parents: %s" % self.initialParents
1560            for b in self.p4BranchesInGit:
1561                if b != "master":
1562
1563                    ## FIXME
1564                    b = b[len(self.projectName):]
1565                self.createdBranches.add(b)
1566
1567        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1568
1569        importProcess = subprocess.Popen(["git", "fast-import"],
1570                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1571                                         stderr=subprocess.PIPE);
1572        self.gitOutput = importProcess.stdout
1573        self.gitStream = importProcess.stdin
1574        self.gitError = importProcess.stderr
1575
1576        if revision:
1577            self.importHeadRevision(revision)
1578        else:
1579            changes = []
1580
1581            if len(self.changesFile) > 0:
1582                output = open(self.changesFile).readlines()
1583                changeSet = Set()
1584                for line in output:
1585                    changeSet.add(int(line))
1586
1587                for change in changeSet:
1588                    changes.append(change)
1589
1590                changes.sort()
1591            else:
1592                if self.verbose:
1593                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1594                                                              self.changeRange)
1595                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1596
1597                if len(self.maxChanges) > 0:
1598                    changes = changes[:min(int(self.maxChanges), len(changes))]
1599
1600            if len(changes) == 0:
1601                if not self.silent:
1602                    print "No changes to import!"
1603                return True
1604
1605            if not self.silent and not self.detectBranches:
1606                print "Import destination: %s" % self.branch
1607
1608            self.updatedBranches = set()
1609
1610            self.importChanges(changes)
1611
1612            if not self.silent:
1613                print ""
1614                if len(self.updatedBranches) > 0:
1615                    sys.stdout.write("Updated branches: ")
1616                    for b in self.updatedBranches:
1617                        sys.stdout.write("%s " % b)
1618                    sys.stdout.write("\n")
1619
1620        self.gitStream.close()
1621        if importProcess.wait() != 0:
1622            die("fast-import failed: %s" % self.gitError.read())
1623        self.gitOutput.close()
1624        self.gitError.close()
1625
1626        return True
1627
1628class P4Rebase(Command):
1629    def __init__(self):
1630        Command.__init__(self)
1631        self.options = [ ]
1632        self.description = ("Fetches the latest revision from perforce and "
1633                            + "rebases the current work (branch) against it")
1634        self.verbose = False
1635
1636    def run(self, args):
1637        sync = P4Sync()
1638        sync.run([])
1639
1640        return self.rebase()
1641
1642    def rebase(self):
1643        [upstream, settings] = findUpstreamBranchPoint()
1644        if len(upstream) == 0:
1645            die("Cannot find upstream branchpoint for rebase")
1646
1647        # the branchpoint may be p4/foo~3, so strip off the parent
1648        upstream = re.sub("~[0-9]+$", "", upstream)
1649
1650        print "Rebasing the current branch onto %s" % upstream
1651        oldHead = read_pipe("git rev-parse HEAD").strip()
1652        system("git rebase %s" % upstream)
1653        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1654        return True
1655
1656class P4Clone(P4Sync):
1657    def __init__(self):
1658        P4Sync.__init__(self)
1659        self.description = "Creates a new git repository and imports from Perforce into it"
1660        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1661        self.options.append(
1662            optparse.make_option("--destination", dest="cloneDestination",
1663                                 action='store', default=None,
1664                                 help="where to leave result of the clone"))
1665        self.cloneDestination = None
1666        self.needsGit = False
1667
1668    def defaultDestination(self, args):
1669        ## TODO: use common prefix of args?
1670        depotPath = args[0]
1671        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1672        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1673        depotDir = re.sub(r"\.\.\.$,", "", depotDir)
1674        depotDir = re.sub(r"/$", "", depotDir)
1675        return os.path.split(depotDir)[1]
1676
1677    def run(self, args):
1678        if len(args) < 1:
1679            return False
1680
1681        if self.keepRepoPath and not self.cloneDestination:
1682            sys.stderr.write("Must specify destination for --keep-path\n")
1683            sys.exit(1)
1684
1685        depotPaths = args
1686
1687        if not self.cloneDestination and len(depotPaths) > 1:
1688            self.cloneDestination = depotPaths[-1]
1689            depotPaths = depotPaths[:-1]
1690
1691        for p in depotPaths:
1692            if not p.startswith("//"):
1693                return False
1694
1695        if not self.cloneDestination:
1696            self.cloneDestination = self.defaultDestination(args)
1697
1698        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1699        if not os.path.exists(self.cloneDestination):
1700            os.makedirs(self.cloneDestination)
1701        os.chdir(self.cloneDestination)
1702        system("git init")
1703        self.gitdir = os.getcwd() + "/.git"
1704        if not P4Sync.run(self, depotPaths):
1705            return False
1706        if self.branch != "master":
1707            if gitBranchExists("refs/remotes/p4/master"):
1708                system("git branch master refs/remotes/p4/master")
1709                system("git checkout -f")
1710            else:
1711                print "Could not detect main branch. No checkout/master branch created."
1712
1713        return True
1714
1715class P4Branches(Command):
1716    def __init__(self):
1717        Command.__init__(self)
1718        self.options = [ ]
1719        self.description = ("Shows the git branches that hold imports and their "
1720                            + "corresponding perforce depot paths")
1721        self.verbose = False
1722
1723    def run(self, args):
1724        if originP4BranchesExist():
1725            createOrUpdateBranchesFromOrigin()
1726
1727        cmdline = "git rev-parse --symbolic "
1728        cmdline += " --remotes"
1729
1730        for line in read_pipe_lines(cmdline):
1731            line = line.strip()
1732
1733            if not line.startswith('p4/') or line == "p4/HEAD":
1734                continue
1735            branch = line
1736
1737            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1738            settings = extractSettingsGitLog(log)
1739
1740            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1741        return True
1742
1743class HelpFormatter(optparse.IndentedHelpFormatter):
1744    def __init__(self):
1745        optparse.IndentedHelpFormatter.__init__(self)
1746
1747    def format_description(self, description):
1748        if description:
1749            return description + "\n"
1750        else:
1751            return ""
1752
1753def printUsage(commands):
1754    print "usage: %s <command> [options]" % sys.argv[0]
1755    print ""
1756    print "valid commands: %s" % ", ".join(commands)
1757    print ""
1758    print "Try %s <command> --help for command specific help." % sys.argv[0]
1759    print ""
1760
1761commands = {
1762    "debug" : P4Debug,
1763    "submit" : P4Submit,
1764    "commit" : P4Submit,
1765    "sync" : P4Sync,
1766    "rebase" : P4Rebase,
1767    "clone" : P4Clone,
1768    "rollback" : P4RollBack,
1769    "branches" : P4Branches
1770}
1771
1772
1773def main():
1774    if len(sys.argv[1:]) == 0:
1775        printUsage(commands.keys())
1776        sys.exit(2)
1777
1778    cmd = ""
1779    cmdName = sys.argv[1]
1780    try:
1781        klass = commands[cmdName]
1782        cmd = klass()
1783    except KeyError:
1784        print "unknown command %s" % cmdName
1785        print ""
1786        printUsage(commands.keys())
1787        sys.exit(2)
1788
1789    options = cmd.options
1790    cmd.gitdir = os.environ.get("GIT_DIR", None)
1791
1792    args = sys.argv[2:]
1793
1794    if len(options) > 0:
1795        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1796
1797        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1798                                       options,
1799                                       description = cmd.description,
1800                                       formatter = HelpFormatter())
1801
1802        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1803    global verbose
1804    verbose = cmd.verbose
1805    if cmd.needsGit:
1806        if cmd.gitdir == None:
1807            cmd.gitdir = os.path.abspath(".git")
1808            if not isValidGitDir(cmd.gitdir):
1809                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1810                if os.path.exists(cmd.gitdir):
1811                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1812                    if len(cdup) > 0:
1813                        os.chdir(cdup);
1814
1815        if not isValidGitDir(cmd.gitdir):
1816            if isValidGitDir(cmd.gitdir + "/.git"):
1817                cmd.gitdir += "/.git"
1818            else:
1819                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1820
1821        os.environ["GIT_DIR"] = cmd.gitdir
1822
1823    if not cmd.run(args):
1824        parser.print_help()
1825
1826
1827if __name__ == '__main__':
1828    main()