git-p4.pyon commit diff-lib: allow ita entries treated as "not yet exist in index" (425a28e)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10import sys
  11if sys.hexversion < 0x02040000:
  12    # The limiter is the subprocess module
  13    sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
  14    sys.exit(1)
  15import os
  16import optparse
  17import marshal
  18import subprocess
  19import tempfile
  20import time
  21import platform
  22import re
  23import shutil
  24import stat
  25import zipfile
  26import zlib
  27import ctypes
  28
  29try:
  30    from subprocess import CalledProcessError
  31except ImportError:
  32    # from python2.7:subprocess.py
  33    # Exception classes used by this module.
  34    class CalledProcessError(Exception):
  35        """This exception is raised when a process run by check_call() returns
  36        a non-zero exit status.  The exit status will be stored in the
  37        returncode attribute."""
  38        def __init__(self, returncode, cmd):
  39            self.returncode = returncode
  40            self.cmd = cmd
  41        def __str__(self):
  42            return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
  43
  44verbose = False
  45
  46# Only labels/tags matching this will be imported/exported
  47defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
  48
  49# Grab changes in blocks of this many revisions, unless otherwise requested
  50defaultBlockSize = 512
  51
  52def p4_build_cmd(cmd):
  53    """Build a suitable p4 command line.
  54
  55    This consolidates building and returning a p4 command line into one
  56    location. It means that hooking into the environment, or other configuration
  57    can be done more easily.
  58    """
  59    real_cmd = ["p4"]
  60
  61    user = gitConfig("git-p4.user")
  62    if len(user) > 0:
  63        real_cmd += ["-u",user]
  64
  65    password = gitConfig("git-p4.password")
  66    if len(password) > 0:
  67        real_cmd += ["-P", password]
  68
  69    port = gitConfig("git-p4.port")
  70    if len(port) > 0:
  71        real_cmd += ["-p", port]
  72
  73    host = gitConfig("git-p4.host")
  74    if len(host) > 0:
  75        real_cmd += ["-H", host]
  76
  77    client = gitConfig("git-p4.client")
  78    if len(client) > 0:
  79        real_cmd += ["-c", client]
  80
  81
  82    if isinstance(cmd,basestring):
  83        real_cmd = ' '.join(real_cmd) + ' ' + cmd
  84    else:
  85        real_cmd += cmd
  86    return real_cmd
  87
  88def chdir(path, is_client_path=False):
  89    """Do chdir to the given path, and set the PWD environment
  90       variable for use by P4.  It does not look at getcwd() output.
  91       Since we're not using the shell, it is necessary to set the
  92       PWD environment variable explicitly.
  93
  94       Normally, expand the path to force it to be absolute.  This
  95       addresses the use of relative path names inside P4 settings,
  96       e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
  97       as given; it looks for .p4config using PWD.
  98
  99       If is_client_path, the path was handed to us directly by p4,
 100       and may be a symbolic link.  Do not call os.getcwd() in this
 101       case, because it will cause p4 to think that PWD is not inside
 102       the client path.
 103       """
 104
 105    os.chdir(path)
 106    if not is_client_path:
 107        path = os.getcwd()
 108    os.environ['PWD'] = path
 109
 110def calcDiskFree():
 111    """Return free space in bytes on the disk of the given dirname."""
 112    if platform.system() == 'Windows':
 113        free_bytes = ctypes.c_ulonglong(0)
 114        ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
 115        return free_bytes.value
 116    else:
 117        st = os.statvfs(os.getcwd())
 118        return st.f_bavail * st.f_frsize
 119
 120def die(msg):
 121    if verbose:
 122        raise Exception(msg)
 123    else:
 124        sys.stderr.write(msg + "\n")
 125        sys.exit(1)
 126
 127def write_pipe(c, stdin):
 128    if verbose:
 129        sys.stderr.write('Writing pipe: %s\n' % str(c))
 130
 131    expand = isinstance(c,basestring)
 132    p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
 133    pipe = p.stdin
 134    val = pipe.write(stdin)
 135    pipe.close()
 136    if p.wait():
 137        die('Command failed: %s' % str(c))
 138
 139    return val
 140
 141def p4_write_pipe(c, stdin):
 142    real_cmd = p4_build_cmd(c)
 143    return write_pipe(real_cmd, stdin)
 144
 145def read_pipe(c, ignore_error=False):
 146    if verbose:
 147        sys.stderr.write('Reading pipe: %s\n' % str(c))
 148
 149    expand = isinstance(c,basestring)
 150    p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
 151    (out, err) = p.communicate()
 152    if p.returncode != 0 and not ignore_error:
 153        die('Command failed: %s\nError: %s' % (str(c), err))
 154    return out
 155
 156def p4_read_pipe(c, ignore_error=False):
 157    real_cmd = p4_build_cmd(c)
 158    return read_pipe(real_cmd, ignore_error)
 159
 160def read_pipe_lines(c):
 161    if verbose:
 162        sys.stderr.write('Reading pipe: %s\n' % str(c))
 163
 164    expand = isinstance(c, basestring)
 165    p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 166    pipe = p.stdout
 167    val = pipe.readlines()
 168    if pipe.close() or p.wait():
 169        die('Command failed: %s' % str(c))
 170
 171    return val
 172
 173def p4_read_pipe_lines(c):
 174    """Specifically invoke p4 on the command supplied. """
 175    real_cmd = p4_build_cmd(c)
 176    return read_pipe_lines(real_cmd)
 177
 178def p4_has_command(cmd):
 179    """Ask p4 for help on this command.  If it returns an error, the
 180       command does not exist in this version of p4."""
 181    real_cmd = p4_build_cmd(["help", cmd])
 182    p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
 183                                   stderr=subprocess.PIPE)
 184    p.communicate()
 185    return p.returncode == 0
 186
 187def p4_has_move_command():
 188    """See if the move command exists, that it supports -k, and that
 189       it has not been administratively disabled.  The arguments
 190       must be correct, but the filenames do not have to exist.  Use
 191       ones with wildcards so even if they exist, it will fail."""
 192
 193    if not p4_has_command("move"):
 194        return False
 195    cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
 196    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 197    (out, err) = p.communicate()
 198    # return code will be 1 in either case
 199    if err.find("Invalid option") >= 0:
 200        return False
 201    if err.find("disabled") >= 0:
 202        return False
 203    # assume it failed because @... was invalid changelist
 204    return True
 205
 206def system(cmd, ignore_error=False):
 207    expand = isinstance(cmd,basestring)
 208    if verbose:
 209        sys.stderr.write("executing %s\n" % str(cmd))
 210    retcode = subprocess.call(cmd, shell=expand)
 211    if retcode and not ignore_error:
 212        raise CalledProcessError(retcode, cmd)
 213
 214    return retcode
 215
 216def p4_system(cmd):
 217    """Specifically invoke p4 as the system command. """
 218    real_cmd = p4_build_cmd(cmd)
 219    expand = isinstance(real_cmd, basestring)
 220    retcode = subprocess.call(real_cmd, shell=expand)
 221    if retcode:
 222        raise CalledProcessError(retcode, real_cmd)
 223
 224_p4_version_string = None
 225def p4_version_string():
 226    """Read the version string, showing just the last line, which
 227       hopefully is the interesting version bit.
 228
 229       $ p4 -V
 230       Perforce - The Fast Software Configuration Management System.
 231       Copyright 1995-2011 Perforce Software.  All rights reserved.
 232       Rev. P4/NTX86/2011.1/393975 (2011/12/16).
 233    """
 234    global _p4_version_string
 235    if not _p4_version_string:
 236        a = p4_read_pipe_lines(["-V"])
 237        _p4_version_string = a[-1].rstrip()
 238    return _p4_version_string
 239
 240def p4_integrate(src, dest):
 241    p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
 242
 243def p4_sync(f, *options):
 244    p4_system(["sync"] + list(options) + [wildcard_encode(f)])
 245
 246def p4_add(f):
 247    # forcibly add file names with wildcards
 248    if wildcard_present(f):
 249        p4_system(["add", "-f", f])
 250    else:
 251        p4_system(["add", f])
 252
 253def p4_delete(f):
 254    p4_system(["delete", wildcard_encode(f)])
 255
 256def p4_edit(f, *options):
 257    p4_system(["edit"] + list(options) + [wildcard_encode(f)])
 258
 259def p4_revert(f):
 260    p4_system(["revert", wildcard_encode(f)])
 261
 262def p4_reopen(type, f):
 263    p4_system(["reopen", "-t", type, wildcard_encode(f)])
 264
 265def p4_move(src, dest):
 266    p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
 267
 268def p4_last_change():
 269    results = p4CmdList(["changes", "-m", "1"])
 270    return int(results[0]['change'])
 271
 272def p4_describe(change):
 273    """Make sure it returns a valid result by checking for
 274       the presence of field "time".  Return a dict of the
 275       results."""
 276
 277    ds = p4CmdList(["describe", "-s", str(change)])
 278    if len(ds) != 1:
 279        die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
 280
 281    d = ds[0]
 282
 283    if "p4ExitCode" in d:
 284        die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
 285                                                      str(d)))
 286    if "code" in d:
 287        if d["code"] == "error":
 288            die("p4 describe -s %d returned error code: %s" % (change, str(d)))
 289
 290    if "time" not in d:
 291        die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
 292
 293    return d
 294
 295#
 296# Canonicalize the p4 type and return a tuple of the
 297# base type, plus any modifiers.  See "p4 help filetypes"
 298# for a list and explanation.
 299#
 300def split_p4_type(p4type):
 301
 302    p4_filetypes_historical = {
 303        "ctempobj": "binary+Sw",
 304        "ctext": "text+C",
 305        "cxtext": "text+Cx",
 306        "ktext": "text+k",
 307        "kxtext": "text+kx",
 308        "ltext": "text+F",
 309        "tempobj": "binary+FSw",
 310        "ubinary": "binary+F",
 311        "uresource": "resource+F",
 312        "uxbinary": "binary+Fx",
 313        "xbinary": "binary+x",
 314        "xltext": "text+Fx",
 315        "xtempobj": "binary+Swx",
 316        "xtext": "text+x",
 317        "xunicode": "unicode+x",
 318        "xutf16": "utf16+x",
 319    }
 320    if p4type in p4_filetypes_historical:
 321        p4type = p4_filetypes_historical[p4type]
 322    mods = ""
 323    s = p4type.split("+")
 324    base = s[0]
 325    mods = ""
 326    if len(s) > 1:
 327        mods = s[1]
 328    return (base, mods)
 329
 330#
 331# return the raw p4 type of a file (text, text+ko, etc)
 332#
 333def p4_type(f):
 334    results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
 335    return results[0]['headType']
 336
 337#
 338# Given a type base and modifier, return a regexp matching
 339# the keywords that can be expanded in the file
 340#
 341def p4_keywords_regexp_for_type(base, type_mods):
 342    if base in ("text", "unicode", "binary"):
 343        kwords = None
 344        if "ko" in type_mods:
 345            kwords = 'Id|Header'
 346        elif "k" in type_mods:
 347            kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
 348        else:
 349            return None
 350        pattern = r"""
 351            \$              # Starts with a dollar, followed by...
 352            (%s)            # one of the keywords, followed by...
 353            (:[^$\n]+)?     # possibly an old expansion, followed by...
 354            \$              # another dollar
 355            """ % kwords
 356        return pattern
 357    else:
 358        return None
 359
 360#
 361# Given a file, return a regexp matching the possible
 362# RCS keywords that will be expanded, or None for files
 363# with kw expansion turned off.
 364#
 365def p4_keywords_regexp_for_file(file):
 366    if not os.path.exists(file):
 367        return None
 368    else:
 369        (type_base, type_mods) = split_p4_type(p4_type(file))
 370        return p4_keywords_regexp_for_type(type_base, type_mods)
 371
 372def setP4ExecBit(file, mode):
 373    # Reopens an already open file and changes the execute bit to match
 374    # the execute bit setting in the passed in mode.
 375
 376    p4Type = "+x"
 377
 378    if not isModeExec(mode):
 379        p4Type = getP4OpenedType(file)
 380        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 381        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 382        if p4Type[-1] == "+":
 383            p4Type = p4Type[0:-1]
 384
 385    p4_reopen(p4Type, file)
 386
 387def getP4OpenedType(file):
 388    # Returns the perforce file type for the given file.
 389
 390    result = p4_read_pipe(["opened", wildcard_encode(file)])
 391    match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
 392    if match:
 393        return match.group(1)
 394    else:
 395        die("Could not determine file type for %s (result: '%s')" % (file, result))
 396
 397# Return the set of all p4 labels
 398def getP4Labels(depotPaths):
 399    labels = set()
 400    if isinstance(depotPaths,basestring):
 401        depotPaths = [depotPaths]
 402
 403    for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
 404        label = l['label']
 405        labels.add(label)
 406
 407    return labels
 408
 409# Return the set of all git tags
 410def getGitTags():
 411    gitTags = set()
 412    for line in read_pipe_lines(["git", "tag"]):
 413        tag = line.strip()
 414        gitTags.add(tag)
 415    return gitTags
 416
 417def diffTreePattern():
 418    # This is a simple generator for the diff tree regex pattern. This could be
 419    # a class variable if this and parseDiffTreeEntry were a part of a class.
 420    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 421    while True:
 422        yield pattern
 423
 424def parseDiffTreeEntry(entry):
 425    """Parses a single diff tree entry into its component elements.
 426
 427    See git-diff-tree(1) manpage for details about the format of the diff
 428    output. This method returns a dictionary with the following elements:
 429
 430    src_mode - The mode of the source file
 431    dst_mode - The mode of the destination file
 432    src_sha1 - The sha1 for the source file
 433    dst_sha1 - The sha1 fr the destination file
 434    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 435    status_score - The score for the status (applicable for 'C' and 'R'
 436                   statuses). This is None if there is no score.
 437    src - The path for the source file.
 438    dst - The path for the destination file. This is only present for
 439          copy or renames. If it is not present, this is None.
 440
 441    If the pattern is not matched, None is returned."""
 442
 443    match = diffTreePattern().next().match(entry)
 444    if match:
 445        return {
 446            'src_mode': match.group(1),
 447            'dst_mode': match.group(2),
 448            'src_sha1': match.group(3),
 449            'dst_sha1': match.group(4),
 450            'status': match.group(5),
 451            'status_score': match.group(6),
 452            'src': match.group(7),
 453            'dst': match.group(10)
 454        }
 455    return None
 456
 457def isModeExec(mode):
 458    # Returns True if the given git mode represents an executable file,
 459    # otherwise False.
 460    return mode[-3:] == "755"
 461
 462def isModeExecChanged(src_mode, dst_mode):
 463    return isModeExec(src_mode) != isModeExec(dst_mode)
 464
 465def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 466
 467    if isinstance(cmd,basestring):
 468        cmd = "-G " + cmd
 469        expand = True
 470    else:
 471        cmd = ["-G"] + cmd
 472        expand = False
 473
 474    cmd = p4_build_cmd(cmd)
 475    if verbose:
 476        sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 477
 478    # Use a temporary file to avoid deadlocks without
 479    # subprocess.communicate(), which would put another copy
 480    # of stdout into memory.
 481    stdin_file = None
 482    if stdin is not None:
 483        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 484        if isinstance(stdin,basestring):
 485            stdin_file.write(stdin)
 486        else:
 487            for i in stdin:
 488                stdin_file.write(i + '\n')
 489        stdin_file.flush()
 490        stdin_file.seek(0)
 491
 492    p4 = subprocess.Popen(cmd,
 493                          shell=expand,
 494                          stdin=stdin_file,
 495                          stdout=subprocess.PIPE)
 496
 497    result = []
 498    try:
 499        while True:
 500            entry = marshal.load(p4.stdout)
 501            if cb is not None:
 502                cb(entry)
 503            else:
 504                result.append(entry)
 505    except EOFError:
 506        pass
 507    exitCode = p4.wait()
 508    if exitCode != 0:
 509        entry = {}
 510        entry["p4ExitCode"] = exitCode
 511        result.append(entry)
 512
 513    return result
 514
 515def p4Cmd(cmd):
 516    list = p4CmdList(cmd)
 517    result = {}
 518    for entry in list:
 519        result.update(entry)
 520    return result;
 521
 522def p4Where(depotPath):
 523    if not depotPath.endswith("/"):
 524        depotPath += "/"
 525    depotPathLong = depotPath + "..."
 526    outputList = p4CmdList(["where", depotPathLong])
 527    output = None
 528    for entry in outputList:
 529        if "depotFile" in entry:
 530            # Search for the base client side depot path, as long as it starts with the branch's P4 path.
 531            # The base path always ends with "/...".
 532            if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
 533                output = entry
 534                break
 535        elif "data" in entry:
 536            data = entry.get("data")
 537            space = data.find(" ")
 538            if data[:space] == depotPath:
 539                output = entry
 540                break
 541    if output == None:
 542        return ""
 543    if output["code"] == "error":
 544        return ""
 545    clientPath = ""
 546    if "path" in output:
 547        clientPath = output.get("path")
 548    elif "data" in output:
 549        data = output.get("data")
 550        lastSpace = data.rfind(" ")
 551        clientPath = data[lastSpace + 1:]
 552
 553    if clientPath.endswith("..."):
 554        clientPath = clientPath[:-3]
 555    return clientPath
 556
 557def currentGitBranch():
 558    retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
 559    if retcode != 0:
 560        # on a detached head
 561        return None
 562    else:
 563        return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
 564
 565def isValidGitDir(path):
 566    if (os.path.exists(path + "/HEAD")
 567        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 568        return True;
 569    return False
 570
 571def parseRevision(ref):
 572    return read_pipe("git rev-parse %s" % ref).strip()
 573
 574def branchExists(ref):
 575    rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 576                     ignore_error=True)
 577    return len(rev) > 0
 578
 579def extractLogMessageFromGitCommit(commit):
 580    logMessage = ""
 581
 582    ## fixme: title is first line of commit, not 1st paragraph.
 583    foundTitle = False
 584    for log in read_pipe_lines("git cat-file commit %s" % commit):
 585       if not foundTitle:
 586           if len(log) == 1:
 587               foundTitle = True
 588           continue
 589
 590       logMessage += log
 591    return logMessage
 592
 593def extractSettingsGitLog(log):
 594    values = {}
 595    for line in log.split("\n"):
 596        line = line.strip()
 597        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 598        if not m:
 599            continue
 600
 601        assignments = m.group(1).split (':')
 602        for a in assignments:
 603            vals = a.split ('=')
 604            key = vals[0].strip()
 605            val = ('='.join (vals[1:])).strip()
 606            if val.endswith ('\"') and val.startswith('"'):
 607                val = val[1:-1]
 608
 609            values[key] = val
 610
 611    paths = values.get("depot-paths")
 612    if not paths:
 613        paths = values.get("depot-path")
 614    if paths:
 615        values['depot-paths'] = paths.split(',')
 616    return values
 617
 618def gitBranchExists(branch):
 619    proc = subprocess.Popen(["git", "rev-parse", branch],
 620                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 621    return proc.wait() == 0;
 622
 623_gitConfig = {}
 624
 625def gitConfig(key, typeSpecifier=None):
 626    if not _gitConfig.has_key(key):
 627        cmd = [ "git", "config" ]
 628        if typeSpecifier:
 629            cmd += [ typeSpecifier ]
 630        cmd += [ key ]
 631        s = read_pipe(cmd, ignore_error=True)
 632        _gitConfig[key] = s.strip()
 633    return _gitConfig[key]
 634
 635def gitConfigBool(key):
 636    """Return a bool, using git config --bool.  It is True only if the
 637       variable is set to true, and False if set to false or not present
 638       in the config."""
 639
 640    if not _gitConfig.has_key(key):
 641        _gitConfig[key] = gitConfig(key, '--bool') == "true"
 642    return _gitConfig[key]
 643
 644def gitConfigInt(key):
 645    if not _gitConfig.has_key(key):
 646        cmd = [ "git", "config", "--int", key ]
 647        s = read_pipe(cmd, ignore_error=True)
 648        v = s.strip()
 649        try:
 650            _gitConfig[key] = int(gitConfig(key, '--int'))
 651        except ValueError:
 652            _gitConfig[key] = None
 653    return _gitConfig[key]
 654
 655def gitConfigList(key):
 656    if not _gitConfig.has_key(key):
 657        s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
 658        _gitConfig[key] = s.strip().split(os.linesep)
 659        if _gitConfig[key] == ['']:
 660            _gitConfig[key] = []
 661    return _gitConfig[key]
 662
 663def p4BranchesInGit(branchesAreInRemotes=True):
 664    """Find all the branches whose names start with "p4/", looking
 665       in remotes or heads as specified by the argument.  Return
 666       a dictionary of { branch: revision } for each one found.
 667       The branch names are the short names, without any
 668       "p4/" prefix."""
 669
 670    branches = {}
 671
 672    cmdline = "git rev-parse --symbolic "
 673    if branchesAreInRemotes:
 674        cmdline += "--remotes"
 675    else:
 676        cmdline += "--branches"
 677
 678    for line in read_pipe_lines(cmdline):
 679        line = line.strip()
 680
 681        # only import to p4/
 682        if not line.startswith('p4/'):
 683            continue
 684        # special symbolic ref to p4/master
 685        if line == "p4/HEAD":
 686            continue
 687
 688        # strip off p4/ prefix
 689        branch = line[len("p4/"):]
 690
 691        branches[branch] = parseRevision(line)
 692
 693    return branches
 694
 695def branch_exists(branch):
 696    """Make sure that the given ref name really exists."""
 697
 698    cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
 699    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 700    out, _ = p.communicate()
 701    if p.returncode:
 702        return False
 703    # expect exactly one line of output: the branch name
 704    return out.rstrip() == branch
 705
 706def findUpstreamBranchPoint(head = "HEAD"):
 707    branches = p4BranchesInGit()
 708    # map from depot-path to branch name
 709    branchByDepotPath = {}
 710    for branch in branches.keys():
 711        tip = branches[branch]
 712        log = extractLogMessageFromGitCommit(tip)
 713        settings = extractSettingsGitLog(log)
 714        if settings.has_key("depot-paths"):
 715            paths = ",".join(settings["depot-paths"])
 716            branchByDepotPath[paths] = "remotes/p4/" + branch
 717
 718    settings = None
 719    parent = 0
 720    while parent < 65535:
 721        commit = head + "~%s" % parent
 722        log = extractLogMessageFromGitCommit(commit)
 723        settings = extractSettingsGitLog(log)
 724        if settings.has_key("depot-paths"):
 725            paths = ",".join(settings["depot-paths"])
 726            if branchByDepotPath.has_key(paths):
 727                return [branchByDepotPath[paths], settings]
 728
 729        parent = parent + 1
 730
 731    return ["", settings]
 732
 733def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 734    if not silent:
 735        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 736               % localRefPrefix)
 737
 738    originPrefix = "origin/p4/"
 739
 740    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 741        line = line.strip()
 742        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 743            continue
 744
 745        headName = line[len(originPrefix):]
 746        remoteHead = localRefPrefix + headName
 747        originHead = line
 748
 749        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 750        if (not original.has_key('depot-paths')
 751            or not original.has_key('change')):
 752            continue
 753
 754        update = False
 755        if not gitBranchExists(remoteHead):
 756            if verbose:
 757                print "creating %s" % remoteHead
 758            update = True
 759        else:
 760            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 761            if settings.has_key('change') > 0:
 762                if settings['depot-paths'] == original['depot-paths']:
 763                    originP4Change = int(original['change'])
 764                    p4Change = int(settings['change'])
 765                    if originP4Change > p4Change:
 766                        print ("%s (%s) is newer than %s (%s). "
 767                               "Updating p4 branch from origin."
 768                               % (originHead, originP4Change,
 769                                  remoteHead, p4Change))
 770                        update = True
 771                else:
 772                    print ("Ignoring: %s was imported from %s while "
 773                           "%s was imported from %s"
 774                           % (originHead, ','.join(original['depot-paths']),
 775                              remoteHead, ','.join(settings['depot-paths'])))
 776
 777        if update:
 778            system("git update-ref %s %s" % (remoteHead, originHead))
 779
 780def originP4BranchesExist():
 781        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 782
 783
 784def p4ParseNumericChangeRange(parts):
 785    changeStart = int(parts[0][1:])
 786    if parts[1] == '#head':
 787        changeEnd = p4_last_change()
 788    else:
 789        changeEnd = int(parts[1])
 790
 791    return (changeStart, changeEnd)
 792
 793def chooseBlockSize(blockSize):
 794    if blockSize:
 795        return blockSize
 796    else:
 797        return defaultBlockSize
 798
 799def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
 800    assert depotPaths
 801
 802    # Parse the change range into start and end. Try to find integer
 803    # revision ranges as these can be broken up into blocks to avoid
 804    # hitting server-side limits (maxrows, maxscanresults). But if
 805    # that doesn't work, fall back to using the raw revision specifier
 806    # strings, without using block mode.
 807
 808    if changeRange is None or changeRange == '':
 809        changeStart = 1
 810        changeEnd = p4_last_change()
 811        block_size = chooseBlockSize(requestedBlockSize)
 812    else:
 813        parts = changeRange.split(',')
 814        assert len(parts) == 2
 815        try:
 816            (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
 817            block_size = chooseBlockSize(requestedBlockSize)
 818        except:
 819            changeStart = parts[0][1:]
 820            changeEnd = parts[1]
 821            if requestedBlockSize:
 822                die("cannot use --changes-block-size with non-numeric revisions")
 823            block_size = None
 824
 825    changes = []
 826
 827    # Retrieve changes a block at a time, to prevent running
 828    # into a MaxResults/MaxScanRows error from the server.
 829
 830    while True:
 831        cmd = ['changes']
 832
 833        if block_size:
 834            end = min(changeEnd, changeStart + block_size)
 835            revisionRange = "%d,%d" % (changeStart, end)
 836        else:
 837            revisionRange = "%s,%s" % (changeStart, changeEnd)
 838
 839        for p in depotPaths:
 840            cmd += ["%s...@%s" % (p, revisionRange)]
 841
 842        # Insert changes in chronological order
 843        for line in reversed(p4_read_pipe_lines(cmd)):
 844            changes.append(int(line.split(" ")[1]))
 845
 846        if not block_size:
 847            break
 848
 849        if end >= changeEnd:
 850            break
 851
 852        changeStart = end + 1
 853
 854    changes = sorted(changes)
 855    return changes
 856
 857def p4PathStartsWith(path, prefix):
 858    # This method tries to remedy a potential mixed-case issue:
 859    #
 860    # If UserA adds  //depot/DirA/file1
 861    # and UserB adds //depot/dira/file2
 862    #
 863    # we may or may not have a problem. If you have core.ignorecase=true,
 864    # we treat DirA and dira as the same directory
 865    if gitConfigBool("core.ignorecase"):
 866        return path.lower().startswith(prefix.lower())
 867    return path.startswith(prefix)
 868
 869def getClientSpec():
 870    """Look at the p4 client spec, create a View() object that contains
 871       all the mappings, and return it."""
 872
 873    specList = p4CmdList("client -o")
 874    if len(specList) != 1:
 875        die('Output from "client -o" is %d lines, expecting 1' %
 876            len(specList))
 877
 878    # dictionary of all client parameters
 879    entry = specList[0]
 880
 881    # the //client/ name
 882    client_name = entry["Client"]
 883
 884    # just the keys that start with "View"
 885    view_keys = [ k for k in entry.keys() if k.startswith("View") ]
 886
 887    # hold this new View
 888    view = View(client_name)
 889
 890    # append the lines, in order, to the view
 891    for view_num in range(len(view_keys)):
 892        k = "View%d" % view_num
 893        if k not in view_keys:
 894            die("Expected view key %s missing" % k)
 895        view.append(entry[k])
 896
 897    return view
 898
 899def getClientRoot():
 900    """Grab the client directory."""
 901
 902    output = p4CmdList("client -o")
 903    if len(output) != 1:
 904        die('Output from "client -o" is %d lines, expecting 1' % len(output))
 905
 906    entry = output[0]
 907    if "Root" not in entry:
 908        die('Client has no "Root"')
 909
 910    return entry["Root"]
 911
 912#
 913# P4 wildcards are not allowed in filenames.  P4 complains
 914# if you simply add them, but you can force it with "-f", in
 915# which case it translates them into %xx encoding internally.
 916#
 917def wildcard_decode(path):
 918    # Search for and fix just these four characters.  Do % last so
 919    # that fixing it does not inadvertently create new %-escapes.
 920    # Cannot have * in a filename in windows; untested as to
 921    # what p4 would do in such a case.
 922    if not platform.system() == "Windows":
 923        path = path.replace("%2A", "*")
 924    path = path.replace("%23", "#") \
 925               .replace("%40", "@") \
 926               .replace("%25", "%")
 927    return path
 928
 929def wildcard_encode(path):
 930    # do % first to avoid double-encoding the %s introduced here
 931    path = path.replace("%", "%25") \
 932               .replace("*", "%2A") \
 933               .replace("#", "%23") \
 934               .replace("@", "%40")
 935    return path
 936
 937def wildcard_present(path):
 938    m = re.search("[*#@%]", path)
 939    return m is not None
 940
 941class LargeFileSystem(object):
 942    """Base class for large file system support."""
 943
 944    def __init__(self, writeToGitStream):
 945        self.largeFiles = set()
 946        self.writeToGitStream = writeToGitStream
 947
 948    def generatePointer(self, cloneDestination, contentFile):
 949        """Return the content of a pointer file that is stored in Git instead of
 950           the actual content."""
 951        assert False, "Method 'generatePointer' required in " + self.__class__.__name__
 952
 953    def pushFile(self, localLargeFile):
 954        """Push the actual content which is not stored in the Git repository to
 955           a server."""
 956        assert False, "Method 'pushFile' required in " + self.__class__.__name__
 957
 958    def hasLargeFileExtension(self, relPath):
 959        return reduce(
 960            lambda a, b: a or b,
 961            [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
 962            False
 963        )
 964
 965    def generateTempFile(self, contents):
 966        contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
 967        for d in contents:
 968            contentFile.write(d)
 969        contentFile.close()
 970        return contentFile.name
 971
 972    def exceedsLargeFileThreshold(self, relPath, contents):
 973        if gitConfigInt('git-p4.largeFileThreshold'):
 974            contentsSize = sum(len(d) for d in contents)
 975            if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
 976                return True
 977        if gitConfigInt('git-p4.largeFileCompressedThreshold'):
 978            contentsSize = sum(len(d) for d in contents)
 979            if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
 980                return False
 981            contentTempFile = self.generateTempFile(contents)
 982            compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
 983            zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
 984            zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
 985            zf.close()
 986            compressedContentsSize = zf.infolist()[0].compress_size
 987            os.remove(contentTempFile)
 988            os.remove(compressedContentFile.name)
 989            if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
 990                return True
 991        return False
 992
 993    def addLargeFile(self, relPath):
 994        self.largeFiles.add(relPath)
 995
 996    def removeLargeFile(self, relPath):
 997        self.largeFiles.remove(relPath)
 998
 999    def isLargeFile(self, relPath):
1000        return relPath in self.largeFiles
1001
1002    def processContent(self, git_mode, relPath, contents):
1003        """Processes the content of git fast import. This method decides if a
1004           file is stored in the large file system and handles all necessary
1005           steps."""
1006        if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1007            contentTempFile = self.generateTempFile(contents)
1008            (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1009
1010            # Move temp file to final location in large file system
1011            largeFileDir = os.path.dirname(localLargeFile)
1012            if not os.path.isdir(largeFileDir):
1013                os.makedirs(largeFileDir)
1014            shutil.move(contentTempFile, localLargeFile)
1015            self.addLargeFile(relPath)
1016            if gitConfigBool('git-p4.largeFilePush'):
1017                self.pushFile(localLargeFile)
1018            if verbose:
1019                sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1020        return (git_mode, contents)
1021
1022class MockLFS(LargeFileSystem):
1023    """Mock large file system for testing."""
1024
1025    def generatePointer(self, contentFile):
1026        """The pointer content is the original content prefixed with "pointer-".
1027           The local filename of the large file storage is derived from the file content.
1028           """
1029        with open(contentFile, 'r') as f:
1030            content = next(f)
1031            gitMode = '100644'
1032            pointerContents = 'pointer-' + content
1033            localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1034            return (gitMode, pointerContents, localLargeFile)
1035
1036    def pushFile(self, localLargeFile):
1037        """The remote filename of the large file storage is the same as the local
1038           one but in a different directory.
1039           """
1040        remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1041        if not os.path.exists(remotePath):
1042            os.makedirs(remotePath)
1043        shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1044
1045class GitLFS(LargeFileSystem):
1046    """Git LFS as backend for the git-p4 large file system.
1047       See https://git-lfs.github.com/ for details."""
1048
1049    def __init__(self, *args):
1050        LargeFileSystem.__init__(self, *args)
1051        self.baseGitAttributes = []
1052
1053    def generatePointer(self, contentFile):
1054        """Generate a Git LFS pointer for the content. Return LFS Pointer file
1055           mode and content which is stored in the Git repository instead of
1056           the actual content. Return also the new location of the actual
1057           content.
1058           """
1059        pointerProcess = subprocess.Popen(
1060            ['git', 'lfs', 'pointer', '--file=' + contentFile],
1061            stdout=subprocess.PIPE
1062        )
1063        pointerFile = pointerProcess.stdout.read()
1064        if pointerProcess.wait():
1065            os.remove(contentFile)
1066            die('git-lfs pointer command failed. Did you install the extension?')
1067
1068        # Git LFS removed the preamble in the output of the 'pointer' command
1069        # starting from version 1.2.0. Check for the preamble here to support
1070        # earlier versions.
1071        # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1072        if pointerFile.startswith('Git LFS pointer for'):
1073            pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1074
1075        oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1076        localLargeFile = os.path.join(
1077            os.getcwd(),
1078            '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1079            oid,
1080        )
1081        # LFS Spec states that pointer files should not have the executable bit set.
1082        gitMode = '100644'
1083        return (gitMode, pointerFile, localLargeFile)
1084
1085    def pushFile(self, localLargeFile):
1086        uploadProcess = subprocess.Popen(
1087            ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1088        )
1089        if uploadProcess.wait():
1090            die('git-lfs push command failed. Did you define a remote?')
1091
1092    def generateGitAttributes(self):
1093        return (
1094            self.baseGitAttributes +
1095            [
1096                '\n',
1097                '#\n',
1098                '# Git LFS (see https://git-lfs.github.com/)\n',
1099                '#\n',
1100            ] +
1101            ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1102                for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1103            ] +
1104            ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1105                for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1106            ]
1107        )
1108
1109    def addLargeFile(self, relPath):
1110        LargeFileSystem.addLargeFile(self, relPath)
1111        self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1112
1113    def removeLargeFile(self, relPath):
1114        LargeFileSystem.removeLargeFile(self, relPath)
1115        self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1116
1117    def processContent(self, git_mode, relPath, contents):
1118        if relPath == '.gitattributes':
1119            self.baseGitAttributes = contents
1120            return (git_mode, self.generateGitAttributes())
1121        else:
1122            return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1123
1124class Command:
1125    def __init__(self):
1126        self.usage = "usage: %prog [options]"
1127        self.needsGit = True
1128        self.verbose = False
1129
1130class P4UserMap:
1131    def __init__(self):
1132        self.userMapFromPerforceServer = False
1133        self.myP4UserId = None
1134
1135    def p4UserId(self):
1136        if self.myP4UserId:
1137            return self.myP4UserId
1138
1139        results = p4CmdList("user -o")
1140        for r in results:
1141            if r.has_key('User'):
1142                self.myP4UserId = r['User']
1143                return r['User']
1144        die("Could not find your p4 user id")
1145
1146    def p4UserIsMe(self, p4User):
1147        # return True if the given p4 user is actually me
1148        me = self.p4UserId()
1149        if not p4User or p4User != me:
1150            return False
1151        else:
1152            return True
1153
1154    def getUserCacheFilename(self):
1155        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1156        return home + "/.gitp4-usercache.txt"
1157
1158    def getUserMapFromPerforceServer(self):
1159        if self.userMapFromPerforceServer:
1160            return
1161        self.users = {}
1162        self.emails = {}
1163
1164        for output in p4CmdList("users"):
1165            if not output.has_key("User"):
1166                continue
1167            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1168            self.emails[output["Email"]] = output["User"]
1169
1170        mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1171        for mapUserConfig in gitConfigList("git-p4.mapUser"):
1172            mapUser = mapUserConfigRegex.findall(mapUserConfig)
1173            if mapUser and len(mapUser[0]) == 3:
1174                user = mapUser[0][0]
1175                fullname = mapUser[0][1]
1176                email = mapUser[0][2]
1177                self.users[user] = fullname + " <" + email + ">"
1178                self.emails[email] = user
1179
1180        s = ''
1181        for (key, val) in self.users.items():
1182            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1183
1184        open(self.getUserCacheFilename(), "wb").write(s)
1185        self.userMapFromPerforceServer = True
1186
1187    def loadUserMapFromCache(self):
1188        self.users = {}
1189        self.userMapFromPerforceServer = False
1190        try:
1191            cache = open(self.getUserCacheFilename(), "rb")
1192            lines = cache.readlines()
1193            cache.close()
1194            for line in lines:
1195                entry = line.strip().split("\t")
1196                self.users[entry[0]] = entry[1]
1197        except IOError:
1198            self.getUserMapFromPerforceServer()
1199
1200class P4Debug(Command):
1201    def __init__(self):
1202        Command.__init__(self)
1203        self.options = []
1204        self.description = "A tool to debug the output of p4 -G."
1205        self.needsGit = False
1206
1207    def run(self, args):
1208        j = 0
1209        for output in p4CmdList(args):
1210            print 'Element: %d' % j
1211            j += 1
1212            print output
1213        return True
1214
1215class P4RollBack(Command):
1216    def __init__(self):
1217        Command.__init__(self)
1218        self.options = [
1219            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1220        ]
1221        self.description = "A tool to debug the multi-branch import. Don't use :)"
1222        self.rollbackLocalBranches = False
1223
1224    def run(self, args):
1225        if len(args) != 1:
1226            return False
1227        maxChange = int(args[0])
1228
1229        if "p4ExitCode" in p4Cmd("changes -m 1"):
1230            die("Problems executing p4");
1231
1232        if self.rollbackLocalBranches:
1233            refPrefix = "refs/heads/"
1234            lines = read_pipe_lines("git rev-parse --symbolic --branches")
1235        else:
1236            refPrefix = "refs/remotes/"
1237            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1238
1239        for line in lines:
1240            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1241                line = line.strip()
1242                ref = refPrefix + line
1243                log = extractLogMessageFromGitCommit(ref)
1244                settings = extractSettingsGitLog(log)
1245
1246                depotPaths = settings['depot-paths']
1247                change = settings['change']
1248
1249                changed = False
1250
1251                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1252                                                           for p in depotPaths]))) == 0:
1253                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1254                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1255                    continue
1256
1257                while change and int(change) > maxChange:
1258                    changed = True
1259                    if self.verbose:
1260                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1261                    system("git update-ref %s \"%s^\"" % (ref, ref))
1262                    log = extractLogMessageFromGitCommit(ref)
1263                    settings =  extractSettingsGitLog(log)
1264
1265
1266                    depotPaths = settings['depot-paths']
1267                    change = settings['change']
1268
1269                if changed:
1270                    print "%s rewound to %s" % (ref, change)
1271
1272        return True
1273
1274class P4Submit(Command, P4UserMap):
1275
1276    conflict_behavior_choices = ("ask", "skip", "quit")
1277
1278    def __init__(self):
1279        Command.__init__(self)
1280        P4UserMap.__init__(self)
1281        self.options = [
1282                optparse.make_option("--origin", dest="origin"),
1283                optparse.make_option("-M", dest="detectRenames", action="store_true"),
1284                # preserve the user, requires relevant p4 permissions
1285                optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1286                optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1287                optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1288                optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1289                optparse.make_option("--conflict", dest="conflict_behavior",
1290                                     choices=self.conflict_behavior_choices),
1291                optparse.make_option("--branch", dest="branch"),
1292        ]
1293        self.description = "Submit changes from git to the perforce depot."
1294        self.usage += " [name of git branch to submit into perforce depot]"
1295        self.origin = ""
1296        self.detectRenames = False
1297        self.preserveUser = gitConfigBool("git-p4.preserveUser")
1298        self.dry_run = False
1299        self.prepare_p4_only = False
1300        self.conflict_behavior = None
1301        self.isWindows = (platform.system() == "Windows")
1302        self.exportLabels = False
1303        self.p4HasMoveCommand = p4_has_move_command()
1304        self.branch = None
1305
1306        if gitConfig('git-p4.largeFileSystem'):
1307            die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1308
1309    def check(self):
1310        if len(p4CmdList("opened ...")) > 0:
1311            die("You have files opened with perforce! Close them before starting the sync.")
1312
1313    def separate_jobs_from_description(self, message):
1314        """Extract and return a possible Jobs field in the commit
1315           message.  It goes into a separate section in the p4 change
1316           specification.
1317
1318           A jobs line starts with "Jobs:" and looks like a new field
1319           in a form.  Values are white-space separated on the same
1320           line or on following lines that start with a tab.
1321
1322           This does not parse and extract the full git commit message
1323           like a p4 form.  It just sees the Jobs: line as a marker
1324           to pass everything from then on directly into the p4 form,
1325           but outside the description section.
1326
1327           Return a tuple (stripped log message, jobs string)."""
1328
1329        m = re.search(r'^Jobs:', message, re.MULTILINE)
1330        if m is None:
1331            return (message, None)
1332
1333        jobtext = message[m.start():]
1334        stripped_message = message[:m.start()].rstrip()
1335        return (stripped_message, jobtext)
1336
1337    def prepareLogMessage(self, template, message, jobs):
1338        """Edits the template returned from "p4 change -o" to insert
1339           the message in the Description field, and the jobs text in
1340           the Jobs field."""
1341        result = ""
1342
1343        inDescriptionSection = False
1344
1345        for line in template.split("\n"):
1346            if line.startswith("#"):
1347                result += line + "\n"
1348                continue
1349
1350            if inDescriptionSection:
1351                if line.startswith("Files:") or line.startswith("Jobs:"):
1352                    inDescriptionSection = False
1353                    # insert Jobs section
1354                    if jobs:
1355                        result += jobs + "\n"
1356                else:
1357                    continue
1358            else:
1359                if line.startswith("Description:"):
1360                    inDescriptionSection = True
1361                    line += "\n"
1362                    for messageLine in message.split("\n"):
1363                        line += "\t" + messageLine + "\n"
1364
1365            result += line + "\n"
1366
1367        return result
1368
1369    def patchRCSKeywords(self, file, pattern):
1370        # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1371        (handle, outFileName) = tempfile.mkstemp(dir='.')
1372        try:
1373            outFile = os.fdopen(handle, "w+")
1374            inFile = open(file, "r")
1375            regexp = re.compile(pattern, re.VERBOSE)
1376            for line in inFile.readlines():
1377                line = regexp.sub(r'$\1$', line)
1378                outFile.write(line)
1379            inFile.close()
1380            outFile.close()
1381            # Forcibly overwrite the original file
1382            os.unlink(file)
1383            shutil.move(outFileName, file)
1384        except:
1385            # cleanup our temporary file
1386            os.unlink(outFileName)
1387            print "Failed to strip RCS keywords in %s" % file
1388            raise
1389
1390        print "Patched up RCS keywords in %s" % file
1391
1392    def p4UserForCommit(self,id):
1393        # Return the tuple (perforce user,git email) for a given git commit id
1394        self.getUserMapFromPerforceServer()
1395        gitEmail = read_pipe(["git", "log", "--max-count=1",
1396                              "--format=%ae", id])
1397        gitEmail = gitEmail.strip()
1398        if not self.emails.has_key(gitEmail):
1399            return (None,gitEmail)
1400        else:
1401            return (self.emails[gitEmail],gitEmail)
1402
1403    def checkValidP4Users(self,commits):
1404        # check if any git authors cannot be mapped to p4 users
1405        for id in commits:
1406            (user,email) = self.p4UserForCommit(id)
1407            if not user:
1408                msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1409                if gitConfigBool("git-p4.allowMissingP4Users"):
1410                    print "%s" % msg
1411                else:
1412                    die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1413
1414    def lastP4Changelist(self):
1415        # Get back the last changelist number submitted in this client spec. This
1416        # then gets used to patch up the username in the change. If the same
1417        # client spec is being used by multiple processes then this might go
1418        # wrong.
1419        results = p4CmdList("client -o")        # find the current client
1420        client = None
1421        for r in results:
1422            if r.has_key('Client'):
1423                client = r['Client']
1424                break
1425        if not client:
1426            die("could not get client spec")
1427        results = p4CmdList(["changes", "-c", client, "-m", "1"])
1428        for r in results:
1429            if r.has_key('change'):
1430                return r['change']
1431        die("Could not get changelist number for last submit - cannot patch up user details")
1432
1433    def modifyChangelistUser(self, changelist, newUser):
1434        # fixup the user field of a changelist after it has been submitted.
1435        changes = p4CmdList("change -o %s" % changelist)
1436        if len(changes) != 1:
1437            die("Bad output from p4 change modifying %s to user %s" %
1438                (changelist, newUser))
1439
1440        c = changes[0]
1441        if c['User'] == newUser: return   # nothing to do
1442        c['User'] = newUser
1443        input = marshal.dumps(c)
1444
1445        result = p4CmdList("change -f -i", stdin=input)
1446        for r in result:
1447            if r.has_key('code'):
1448                if r['code'] == 'error':
1449                    die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1450            if r.has_key('data'):
1451                print("Updated user field for changelist %s to %s" % (changelist, newUser))
1452                return
1453        die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1454
1455    def canChangeChangelists(self):
1456        # check to see if we have p4 admin or super-user permissions, either of
1457        # which are required to modify changelists.
1458        results = p4CmdList(["protects", self.depotPath])
1459        for r in results:
1460            if r.has_key('perm'):
1461                if r['perm'] == 'admin':
1462                    return 1
1463                if r['perm'] == 'super':
1464                    return 1
1465        return 0
1466
1467    def prepareSubmitTemplate(self):
1468        """Run "p4 change -o" to grab a change specification template.
1469           This does not use "p4 -G", as it is nice to keep the submission
1470           template in original order, since a human might edit it.
1471
1472           Remove lines in the Files section that show changes to files
1473           outside the depot path we're committing into."""
1474
1475        [upstream, settings] = findUpstreamBranchPoint()
1476
1477        template = ""
1478        inFilesSection = False
1479        for line in p4_read_pipe_lines(['change', '-o']):
1480            if line.endswith("\r\n"):
1481                line = line[:-2] + "\n"
1482            if inFilesSection:
1483                if line.startswith("\t"):
1484                    # path starts and ends with a tab
1485                    path = line[1:]
1486                    lastTab = path.rfind("\t")
1487                    if lastTab != -1:
1488                        path = path[:lastTab]
1489                        if settings.has_key('depot-paths'):
1490                            if not [p for p in settings['depot-paths']
1491                                    if p4PathStartsWith(path, p)]:
1492                                continue
1493                        else:
1494                            if not p4PathStartsWith(path, self.depotPath):
1495                                continue
1496                else:
1497                    inFilesSection = False
1498            else:
1499                if line.startswith("Files:"):
1500                    inFilesSection = True
1501
1502            template += line
1503
1504        return template
1505
1506    def edit_template(self, template_file):
1507        """Invoke the editor to let the user change the submission
1508           message.  Return true if okay to continue with the submit."""
1509
1510        # if configured to skip the editing part, just submit
1511        if gitConfigBool("git-p4.skipSubmitEdit"):
1512            return True
1513
1514        # look at the modification time, to check later if the user saved
1515        # the file
1516        mtime = os.stat(template_file).st_mtime
1517
1518        # invoke the editor
1519        if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1520            editor = os.environ.get("P4EDITOR")
1521        else:
1522            editor = read_pipe("git var GIT_EDITOR").strip()
1523        system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1524
1525        # If the file was not saved, prompt to see if this patch should
1526        # be skipped.  But skip this verification step if configured so.
1527        if gitConfigBool("git-p4.skipSubmitEditCheck"):
1528            return True
1529
1530        # modification time updated means user saved the file
1531        if os.stat(template_file).st_mtime > mtime:
1532            return True
1533
1534        while True:
1535            response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1536            if response == 'y':
1537                return True
1538            if response == 'n':
1539                return False
1540
1541    def get_diff_description(self, editedFiles, filesToAdd):
1542        # diff
1543        if os.environ.has_key("P4DIFF"):
1544            del(os.environ["P4DIFF"])
1545        diff = ""
1546        for editedFile in editedFiles:
1547            diff += p4_read_pipe(['diff', '-du',
1548                                  wildcard_encode(editedFile)])
1549
1550        # new file diff
1551        newdiff = ""
1552        for newFile in filesToAdd:
1553            newdiff += "==== new file ====\n"
1554            newdiff += "--- /dev/null\n"
1555            newdiff += "+++ %s\n" % newFile
1556            f = open(newFile, "r")
1557            for line in f.readlines():
1558                newdiff += "+" + line
1559            f.close()
1560
1561        return (diff + newdiff).replace('\r\n', '\n')
1562
1563    def applyCommit(self, id):
1564        """Apply one commit, return True if it succeeded."""
1565
1566        print "Applying", read_pipe(["git", "show", "-s",
1567                                     "--format=format:%h %s", id])
1568
1569        (p4User, gitEmail) = self.p4UserForCommit(id)
1570
1571        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1572        filesToAdd = set()
1573        filesToChangeType = set()
1574        filesToDelete = set()
1575        editedFiles = set()
1576        pureRenameCopy = set()
1577        filesToChangeExecBit = {}
1578
1579        for line in diff:
1580            diff = parseDiffTreeEntry(line)
1581            modifier = diff['status']
1582            path = diff['src']
1583            if modifier == "M":
1584                p4_edit(path)
1585                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1586                    filesToChangeExecBit[path] = diff['dst_mode']
1587                editedFiles.add(path)
1588            elif modifier == "A":
1589                filesToAdd.add(path)
1590                filesToChangeExecBit[path] = diff['dst_mode']
1591                if path in filesToDelete:
1592                    filesToDelete.remove(path)
1593            elif modifier == "D":
1594                filesToDelete.add(path)
1595                if path in filesToAdd:
1596                    filesToAdd.remove(path)
1597            elif modifier == "C":
1598                src, dest = diff['src'], diff['dst']
1599                p4_integrate(src, dest)
1600                pureRenameCopy.add(dest)
1601                if diff['src_sha1'] != diff['dst_sha1']:
1602                    p4_edit(dest)
1603                    pureRenameCopy.discard(dest)
1604                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1605                    p4_edit(dest)
1606                    pureRenameCopy.discard(dest)
1607                    filesToChangeExecBit[dest] = diff['dst_mode']
1608                if self.isWindows:
1609                    # turn off read-only attribute
1610                    os.chmod(dest, stat.S_IWRITE)
1611                os.unlink(dest)
1612                editedFiles.add(dest)
1613            elif modifier == "R":
1614                src, dest = diff['src'], diff['dst']
1615                if self.p4HasMoveCommand:
1616                    p4_edit(src)        # src must be open before move
1617                    p4_move(src, dest)  # opens for (move/delete, move/add)
1618                else:
1619                    p4_integrate(src, dest)
1620                    if diff['src_sha1'] != diff['dst_sha1']:
1621                        p4_edit(dest)
1622                    else:
1623                        pureRenameCopy.add(dest)
1624                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1625                    if not self.p4HasMoveCommand:
1626                        p4_edit(dest)   # with move: already open, writable
1627                    filesToChangeExecBit[dest] = diff['dst_mode']
1628                if not self.p4HasMoveCommand:
1629                    if self.isWindows:
1630                        os.chmod(dest, stat.S_IWRITE)
1631                    os.unlink(dest)
1632                    filesToDelete.add(src)
1633                editedFiles.add(dest)
1634            elif modifier == "T":
1635                filesToChangeType.add(path)
1636            else:
1637                die("unknown modifier %s for %s" % (modifier, path))
1638
1639        diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1640        patchcmd = diffcmd + " | git apply "
1641        tryPatchCmd = patchcmd + "--check -"
1642        applyPatchCmd = patchcmd + "--check --apply -"
1643        patch_succeeded = True
1644
1645        if os.system(tryPatchCmd) != 0:
1646            fixed_rcs_keywords = False
1647            patch_succeeded = False
1648            print "Unfortunately applying the change failed!"
1649
1650            # Patch failed, maybe it's just RCS keyword woes. Look through
1651            # the patch to see if that's possible.
1652            if gitConfigBool("git-p4.attemptRCSCleanup"):
1653                file = None
1654                pattern = None
1655                kwfiles = {}
1656                for file in editedFiles | filesToDelete:
1657                    # did this file's delta contain RCS keywords?
1658                    pattern = p4_keywords_regexp_for_file(file)
1659
1660                    if pattern:
1661                        # this file is a possibility...look for RCS keywords.
1662                        regexp = re.compile(pattern, re.VERBOSE)
1663                        for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1664                            if regexp.search(line):
1665                                if verbose:
1666                                    print "got keyword match on %s in %s in %s" % (pattern, line, file)
1667                                kwfiles[file] = pattern
1668                                break
1669
1670                for file in kwfiles:
1671                    if verbose:
1672                        print "zapping %s with %s" % (line,pattern)
1673                    # File is being deleted, so not open in p4.  Must
1674                    # disable the read-only bit on windows.
1675                    if self.isWindows and file not in editedFiles:
1676                        os.chmod(file, stat.S_IWRITE)
1677                    self.patchRCSKeywords(file, kwfiles[file])
1678                    fixed_rcs_keywords = True
1679
1680            if fixed_rcs_keywords:
1681                print "Retrying the patch with RCS keywords cleaned up"
1682                if os.system(tryPatchCmd) == 0:
1683                    patch_succeeded = True
1684
1685        if not patch_succeeded:
1686            for f in editedFiles:
1687                p4_revert(f)
1688            return False
1689
1690        #
1691        # Apply the patch for real, and do add/delete/+x handling.
1692        #
1693        system(applyPatchCmd)
1694
1695        for f in filesToChangeType:
1696            p4_edit(f, "-t", "auto")
1697        for f in filesToAdd:
1698            p4_add(f)
1699        for f in filesToDelete:
1700            p4_revert(f)
1701            p4_delete(f)
1702
1703        # Set/clear executable bits
1704        for f in filesToChangeExecBit.keys():
1705            mode = filesToChangeExecBit[f]
1706            setP4ExecBit(f, mode)
1707
1708        #
1709        # Build p4 change description, starting with the contents
1710        # of the git commit message.
1711        #
1712        logMessage = extractLogMessageFromGitCommit(id)
1713        logMessage = logMessage.strip()
1714        (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1715
1716        template = self.prepareSubmitTemplate()
1717        submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1718
1719        if self.preserveUser:
1720           submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1721
1722        if self.checkAuthorship and not self.p4UserIsMe(p4User):
1723            submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1724            submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1725            submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1726
1727        separatorLine = "######## everything below this line is just the diff #######\n"
1728        if not self.prepare_p4_only:
1729            submitTemplate += separatorLine
1730            submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1731
1732        (handle, fileName) = tempfile.mkstemp()
1733        tmpFile = os.fdopen(handle, "w+b")
1734        if self.isWindows:
1735            submitTemplate = submitTemplate.replace("\n", "\r\n")
1736        tmpFile.write(submitTemplate)
1737        tmpFile.close()
1738
1739        if self.prepare_p4_only:
1740            #
1741            # Leave the p4 tree prepared, and the submit template around
1742            # and let the user decide what to do next
1743            #
1744            print
1745            print "P4 workspace prepared for submission."
1746            print "To submit or revert, go to client workspace"
1747            print "  " + self.clientPath
1748            print
1749            print "To submit, use \"p4 submit\" to write a new description,"
1750            print "or \"p4 submit -i <%s\" to use the one prepared by" \
1751                  " \"git p4\"." % fileName
1752            print "You can delete the file \"%s\" when finished." % fileName
1753
1754            if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1755                print "To preserve change ownership by user %s, you must\n" \
1756                      "do \"p4 change -f <change>\" after submitting and\n" \
1757                      "edit the User field."
1758            if pureRenameCopy:
1759                print "After submitting, renamed files must be re-synced."
1760                print "Invoke \"p4 sync -f\" on each of these files:"
1761                for f in pureRenameCopy:
1762                    print "  " + f
1763
1764            print
1765            print "To revert the changes, use \"p4 revert ...\", and delete"
1766            print "the submit template file \"%s\"" % fileName
1767            if filesToAdd:
1768                print "Since the commit adds new files, they must be deleted:"
1769                for f in filesToAdd:
1770                    print "  " + f
1771            print
1772            return True
1773
1774        #
1775        # Let the user edit the change description, then submit it.
1776        #
1777        submitted = False
1778
1779        try:
1780            if self.edit_template(fileName):
1781                # read the edited message and submit
1782                tmpFile = open(fileName, "rb")
1783                message = tmpFile.read()
1784                tmpFile.close()
1785                if self.isWindows:
1786                    message = message.replace("\r\n", "\n")
1787                submitTemplate = message[:message.index(separatorLine)]
1788                p4_write_pipe(['submit', '-i'], submitTemplate)
1789
1790                if self.preserveUser:
1791                    if p4User:
1792                        # Get last changelist number. Cannot easily get it from
1793                        # the submit command output as the output is
1794                        # unmarshalled.
1795                        changelist = self.lastP4Changelist()
1796                        self.modifyChangelistUser(changelist, p4User)
1797
1798                # The rename/copy happened by applying a patch that created a
1799                # new file.  This leaves it writable, which confuses p4.
1800                for f in pureRenameCopy:
1801                    p4_sync(f, "-f")
1802                submitted = True
1803
1804        finally:
1805            # skip this patch
1806            if not submitted:
1807                print "Submission cancelled, undoing p4 changes."
1808                for f in editedFiles:
1809                    p4_revert(f)
1810                for f in filesToAdd:
1811                    p4_revert(f)
1812                    os.remove(f)
1813                for f in filesToDelete:
1814                    p4_revert(f)
1815
1816        os.remove(fileName)
1817        return submitted
1818
1819    # Export git tags as p4 labels. Create a p4 label and then tag
1820    # with that.
1821    def exportGitTags(self, gitTags):
1822        validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1823        if len(validLabelRegexp) == 0:
1824            validLabelRegexp = defaultLabelRegexp
1825        m = re.compile(validLabelRegexp)
1826
1827        for name in gitTags:
1828
1829            if not m.match(name):
1830                if verbose:
1831                    print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1832                continue
1833
1834            # Get the p4 commit this corresponds to
1835            logMessage = extractLogMessageFromGitCommit(name)
1836            values = extractSettingsGitLog(logMessage)
1837
1838            if not values.has_key('change'):
1839                # a tag pointing to something not sent to p4; ignore
1840                if verbose:
1841                    print "git tag %s does not give a p4 commit" % name
1842                continue
1843            else:
1844                changelist = values['change']
1845
1846            # Get the tag details.
1847            inHeader = True
1848            isAnnotated = False
1849            body = []
1850            for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1851                l = l.strip()
1852                if inHeader:
1853                    if re.match(r'tag\s+', l):
1854                        isAnnotated = True
1855                    elif re.match(r'\s*$', l):
1856                        inHeader = False
1857                        continue
1858                else:
1859                    body.append(l)
1860
1861            if not isAnnotated:
1862                body = ["lightweight tag imported by git p4\n"]
1863
1864            # Create the label - use the same view as the client spec we are using
1865            clientSpec = getClientSpec()
1866
1867            labelTemplate  = "Label: %s\n" % name
1868            labelTemplate += "Description:\n"
1869            for b in body:
1870                labelTemplate += "\t" + b + "\n"
1871            labelTemplate += "View:\n"
1872            for depot_side in clientSpec.mappings:
1873                labelTemplate += "\t%s\n" % depot_side
1874
1875            if self.dry_run:
1876                print "Would create p4 label %s for tag" % name
1877            elif self.prepare_p4_only:
1878                print "Not creating p4 label %s for tag due to option" \
1879                      " --prepare-p4-only" % name
1880            else:
1881                p4_write_pipe(["label", "-i"], labelTemplate)
1882
1883                # Use the label
1884                p4_system(["tag", "-l", name] +
1885                          ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1886
1887                if verbose:
1888                    print "created p4 label for tag %s" % name
1889
1890    def run(self, args):
1891        if len(args) == 0:
1892            self.master = currentGitBranch()
1893        elif len(args) == 1:
1894            self.master = args[0]
1895            if not branchExists(self.master):
1896                die("Branch %s does not exist" % self.master)
1897        else:
1898            return False
1899
1900        if self.master:
1901            allowSubmit = gitConfig("git-p4.allowSubmit")
1902            if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1903                die("%s is not in git-p4.allowSubmit" % self.master)
1904
1905        [upstream, settings] = findUpstreamBranchPoint()
1906        self.depotPath = settings['depot-paths'][0]
1907        if len(self.origin) == 0:
1908            self.origin = upstream
1909
1910        if self.preserveUser:
1911            if not self.canChangeChangelists():
1912                die("Cannot preserve user names without p4 super-user or admin permissions")
1913
1914        # if not set from the command line, try the config file
1915        if self.conflict_behavior is None:
1916            val = gitConfig("git-p4.conflict")
1917            if val:
1918                if val not in self.conflict_behavior_choices:
1919                    die("Invalid value '%s' for config git-p4.conflict" % val)
1920            else:
1921                val = "ask"
1922            self.conflict_behavior = val
1923
1924        if self.verbose:
1925            print "Origin branch is " + self.origin
1926
1927        if len(self.depotPath) == 0:
1928            print "Internal error: cannot locate perforce depot path from existing branches"
1929            sys.exit(128)
1930
1931        self.useClientSpec = False
1932        if gitConfigBool("git-p4.useclientspec"):
1933            self.useClientSpec = True
1934        if self.useClientSpec:
1935            self.clientSpecDirs = getClientSpec()
1936
1937        # Check for the existence of P4 branches
1938        branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1939
1940        if self.useClientSpec and not branchesDetected:
1941            # all files are relative to the client spec
1942            self.clientPath = getClientRoot()
1943        else:
1944            self.clientPath = p4Where(self.depotPath)
1945
1946        if self.clientPath == "":
1947            die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1948
1949        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1950        self.oldWorkingDirectory = os.getcwd()
1951
1952        # ensure the clientPath exists
1953        new_client_dir = False
1954        if not os.path.exists(self.clientPath):
1955            new_client_dir = True
1956            os.makedirs(self.clientPath)
1957
1958        chdir(self.clientPath, is_client_path=True)
1959        if self.dry_run:
1960            print "Would synchronize p4 checkout in %s" % self.clientPath
1961        else:
1962            print "Synchronizing p4 checkout..."
1963            if new_client_dir:
1964                # old one was destroyed, and maybe nobody told p4
1965                p4_sync("...", "-f")
1966            else:
1967                p4_sync("...")
1968        self.check()
1969
1970        commits = []
1971        if self.master:
1972            commitish = self.master
1973        else:
1974            commitish = 'HEAD'
1975
1976        for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
1977            commits.append(line.strip())
1978        commits.reverse()
1979
1980        if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1981            self.checkAuthorship = False
1982        else:
1983            self.checkAuthorship = True
1984
1985        if self.preserveUser:
1986            self.checkValidP4Users(commits)
1987
1988        #
1989        # Build up a set of options to be passed to diff when
1990        # submitting each commit to p4.
1991        #
1992        if self.detectRenames:
1993            # command-line -M arg
1994            self.diffOpts = "-M"
1995        else:
1996            # If not explicitly set check the config variable
1997            detectRenames = gitConfig("git-p4.detectRenames")
1998
1999            if detectRenames.lower() == "false" or detectRenames == "":
2000                self.diffOpts = ""
2001            elif detectRenames.lower() == "true":
2002                self.diffOpts = "-M"
2003            else:
2004                self.diffOpts = "-M%s" % detectRenames
2005
2006        # no command-line arg for -C or --find-copies-harder, just
2007        # config variables
2008        detectCopies = gitConfig("git-p4.detectCopies")
2009        if detectCopies.lower() == "false" or detectCopies == "":
2010            pass
2011        elif detectCopies.lower() == "true":
2012            self.diffOpts += " -C"
2013        else:
2014            self.diffOpts += " -C%s" % detectCopies
2015
2016        if gitConfigBool("git-p4.detectCopiesHarder"):
2017            self.diffOpts += " --find-copies-harder"
2018
2019        #
2020        # Apply the commits, one at a time.  On failure, ask if should
2021        # continue to try the rest of the patches, or quit.
2022        #
2023        if self.dry_run:
2024            print "Would apply"
2025        applied = []
2026        last = len(commits) - 1
2027        for i, commit in enumerate(commits):
2028            if self.dry_run:
2029                print " ", read_pipe(["git", "show", "-s",
2030                                      "--format=format:%h %s", commit])
2031                ok = True
2032            else:
2033                ok = self.applyCommit(commit)
2034            if ok:
2035                applied.append(commit)
2036            else:
2037                if self.prepare_p4_only and i < last:
2038                    print "Processing only the first commit due to option" \
2039                          " --prepare-p4-only"
2040                    break
2041                if i < last:
2042                    quit = False
2043                    while True:
2044                        # prompt for what to do, or use the option/variable
2045                        if self.conflict_behavior == "ask":
2046                            print "What do you want to do?"
2047                            response = raw_input("[s]kip this commit but apply"
2048                                                 " the rest, or [q]uit? ")
2049                            if not response:
2050                                continue
2051                        elif self.conflict_behavior == "skip":
2052                            response = "s"
2053                        elif self.conflict_behavior == "quit":
2054                            response = "q"
2055                        else:
2056                            die("Unknown conflict_behavior '%s'" %
2057                                self.conflict_behavior)
2058
2059                        if response[0] == "s":
2060                            print "Skipping this commit, but applying the rest"
2061                            break
2062                        if response[0] == "q":
2063                            print "Quitting"
2064                            quit = True
2065                            break
2066                    if quit:
2067                        break
2068
2069        chdir(self.oldWorkingDirectory)
2070
2071        if self.dry_run:
2072            pass
2073        elif self.prepare_p4_only:
2074            pass
2075        elif len(commits) == len(applied):
2076            print "All commits applied!"
2077
2078            sync = P4Sync()
2079            if self.branch:
2080                sync.branch = self.branch
2081            sync.run([])
2082
2083            rebase = P4Rebase()
2084            rebase.rebase()
2085
2086        else:
2087            if len(applied) == 0:
2088                print "No commits applied."
2089            else:
2090                print "Applied only the commits marked with '*':"
2091                for c in commits:
2092                    if c in applied:
2093                        star = "*"
2094                    else:
2095                        star = " "
2096                    print star, read_pipe(["git", "show", "-s",
2097                                           "--format=format:%h %s",  c])
2098                print "You will have to do 'git p4 sync' and rebase."
2099
2100        if gitConfigBool("git-p4.exportLabels"):
2101            self.exportLabels = True
2102
2103        if self.exportLabels:
2104            p4Labels = getP4Labels(self.depotPath)
2105            gitTags = getGitTags()
2106
2107            missingGitTags = gitTags - p4Labels
2108            self.exportGitTags(missingGitTags)
2109
2110        # exit with error unless everything applied perfectly
2111        if len(commits) != len(applied):
2112                sys.exit(1)
2113
2114        return True
2115
2116class View(object):
2117    """Represent a p4 view ("p4 help views"), and map files in a
2118       repo according to the view."""
2119
2120    def __init__(self, client_name):
2121        self.mappings = []
2122        self.client_prefix = "//%s/" % client_name
2123        # cache results of "p4 where" to lookup client file locations
2124        self.client_spec_path_cache = {}
2125
2126    def append(self, view_line):
2127        """Parse a view line, splitting it into depot and client
2128           sides.  Append to self.mappings, preserving order.  This
2129           is only needed for tag creation."""
2130
2131        # Split the view line into exactly two words.  P4 enforces
2132        # structure on these lines that simplifies this quite a bit.
2133        #
2134        # Either or both words may be double-quoted.
2135        # Single quotes do not matter.
2136        # Double-quote marks cannot occur inside the words.
2137        # A + or - prefix is also inside the quotes.
2138        # There are no quotes unless they contain a space.
2139        # The line is already white-space stripped.
2140        # The two words are separated by a single space.
2141        #
2142        if view_line[0] == '"':
2143            # First word is double quoted.  Find its end.
2144            close_quote_index = view_line.find('"', 1)
2145            if close_quote_index <= 0:
2146                die("No first-word closing quote found: %s" % view_line)
2147            depot_side = view_line[1:close_quote_index]
2148            # skip closing quote and space
2149            rhs_index = close_quote_index + 1 + 1
2150        else:
2151            space_index = view_line.find(" ")
2152            if space_index <= 0:
2153                die("No word-splitting space found: %s" % view_line)
2154            depot_side = view_line[0:space_index]
2155            rhs_index = space_index + 1
2156
2157        # prefix + means overlay on previous mapping
2158        if depot_side.startswith("+"):
2159            depot_side = depot_side[1:]
2160
2161        # prefix - means exclude this path, leave out of mappings
2162        exclude = False
2163        if depot_side.startswith("-"):
2164            exclude = True
2165            depot_side = depot_side[1:]
2166
2167        if not exclude:
2168            self.mappings.append(depot_side)
2169
2170    def convert_client_path(self, clientFile):
2171        # chop off //client/ part to make it relative
2172        if not clientFile.startswith(self.client_prefix):
2173            die("No prefix '%s' on clientFile '%s'" %
2174                (self.client_prefix, clientFile))
2175        return clientFile[len(self.client_prefix):]
2176
2177    def update_client_spec_path_cache(self, files):
2178        """ Caching file paths by "p4 where" batch query """
2179
2180        # List depot file paths exclude that already cached
2181        fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2182
2183        if len(fileArgs) == 0:
2184            return  # All files in cache
2185
2186        where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2187        for res in where_result:
2188            if "code" in res and res["code"] == "error":
2189                # assume error is "... file(s) not in client view"
2190                continue
2191            if "clientFile" not in res:
2192                die("No clientFile in 'p4 where' output")
2193            if "unmap" in res:
2194                # it will list all of them, but only one not unmap-ped
2195                continue
2196            if gitConfigBool("core.ignorecase"):
2197                res['depotFile'] = res['depotFile'].lower()
2198            self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2199
2200        # not found files or unmap files set to ""
2201        for depotFile in fileArgs:
2202            if gitConfigBool("core.ignorecase"):
2203                depotFile = depotFile.lower()
2204            if depotFile not in self.client_spec_path_cache:
2205                self.client_spec_path_cache[depotFile] = ""
2206
2207    def map_in_client(self, depot_path):
2208        """Return the relative location in the client where this
2209           depot file should live.  Returns "" if the file should
2210           not be mapped in the client."""
2211
2212        if gitConfigBool("core.ignorecase"):
2213            depot_path = depot_path.lower()
2214
2215        if depot_path in self.client_spec_path_cache:
2216            return self.client_spec_path_cache[depot_path]
2217
2218        die( "Error: %s is not found in client spec path" % depot_path )
2219        return ""
2220
2221class P4Sync(Command, P4UserMap):
2222    delete_actions = ( "delete", "move/delete", "purge" )
2223
2224    def __init__(self):
2225        Command.__init__(self)
2226        P4UserMap.__init__(self)
2227        self.options = [
2228                optparse.make_option("--branch", dest="branch"),
2229                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2230                optparse.make_option("--changesfile", dest="changesFile"),
2231                optparse.make_option("--silent", dest="silent", action="store_true"),
2232                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2233                optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2234                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2235                                     help="Import into refs/heads/ , not refs/remotes"),
2236                optparse.make_option("--max-changes", dest="maxChanges",
2237                                     help="Maximum number of changes to import"),
2238                optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2239                                     help="Internal block size to use when iteratively calling p4 changes"),
2240                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2241                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2242                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2243                                     help="Only sync files that are included in the Perforce Client Spec"),
2244                optparse.make_option("-/", dest="cloneExclude",
2245                                     action="append", type="string",
2246                                     help="exclude depot path"),
2247        ]
2248        self.description = """Imports from Perforce into a git repository.\n
2249    example:
2250    //depot/my/project/ -- to import the current head
2251    //depot/my/project/@all -- to import everything
2252    //depot/my/project/@1,6 -- to import only from revision 1 to 6
2253
2254    (a ... is not needed in the path p4 specification, it's added implicitly)"""
2255
2256        self.usage += " //depot/path[@revRange]"
2257        self.silent = False
2258        self.createdBranches = set()
2259        self.committedChanges = set()
2260        self.branch = ""
2261        self.detectBranches = False
2262        self.detectLabels = False
2263        self.importLabels = False
2264        self.changesFile = ""
2265        self.syncWithOrigin = True
2266        self.importIntoRemotes = True
2267        self.maxChanges = ""
2268        self.changes_block_size = None
2269        self.keepRepoPath = False
2270        self.depotPaths = None
2271        self.p4BranchesInGit = []
2272        self.cloneExclude = []
2273        self.useClientSpec = False
2274        self.useClientSpec_from_options = False
2275        self.clientSpecDirs = None
2276        self.tempBranches = []
2277        self.tempBranchLocation = "refs/git-p4-tmp"
2278        self.largeFileSystem = None
2279
2280        if gitConfig('git-p4.largeFileSystem'):
2281            largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2282            self.largeFileSystem = largeFileSystemConstructor(
2283                lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2284            )
2285
2286        if gitConfig("git-p4.syncFromOrigin") == "false":
2287            self.syncWithOrigin = False
2288
2289    # This is required for the "append" cloneExclude action
2290    def ensure_value(self, attr, value):
2291        if not hasattr(self, attr) or getattr(self, attr) is None:
2292            setattr(self, attr, value)
2293        return getattr(self, attr)
2294
2295    # Force a checkpoint in fast-import and wait for it to finish
2296    def checkpoint(self):
2297        self.gitStream.write("checkpoint\n\n")
2298        self.gitStream.write("progress checkpoint\n\n")
2299        out = self.gitOutput.readline()
2300        if self.verbose:
2301            print "checkpoint finished: " + out
2302
2303    def extractFilesFromCommit(self, commit):
2304        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2305                             for path in self.cloneExclude]
2306        files = []
2307        fnum = 0
2308        while commit.has_key("depotFile%s" % fnum):
2309            path =  commit["depotFile%s" % fnum]
2310
2311            if [p for p in self.cloneExclude
2312                if p4PathStartsWith(path, p)]:
2313                found = False
2314            else:
2315                found = [p for p in self.depotPaths
2316                         if p4PathStartsWith(path, p)]
2317            if not found:
2318                fnum = fnum + 1
2319                continue
2320
2321            file = {}
2322            file["path"] = path
2323            file["rev"] = commit["rev%s" % fnum]
2324            file["action"] = commit["action%s" % fnum]
2325            file["type"] = commit["type%s" % fnum]
2326            files.append(file)
2327            fnum = fnum + 1
2328        return files
2329
2330    def extractJobsFromCommit(self, commit):
2331        jobs = []
2332        jnum = 0
2333        while commit.has_key("job%s" % jnum):
2334            job = commit["job%s" % jnum]
2335            jobs.append(job)
2336            jnum = jnum + 1
2337        return jobs
2338
2339    def stripRepoPath(self, path, prefixes):
2340        """When streaming files, this is called to map a p4 depot path
2341           to where it should go in git.  The prefixes are either
2342           self.depotPaths, or self.branchPrefixes in the case of
2343           branch detection."""
2344
2345        if self.useClientSpec:
2346            # branch detection moves files up a level (the branch name)
2347            # from what client spec interpretation gives
2348            path = self.clientSpecDirs.map_in_client(path)
2349            if self.detectBranches:
2350                for b in self.knownBranches:
2351                    if path.startswith(b + "/"):
2352                        path = path[len(b)+1:]
2353
2354        elif self.keepRepoPath:
2355            # Preserve everything in relative path name except leading
2356            # //depot/; just look at first prefix as they all should
2357            # be in the same depot.
2358            depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2359            if p4PathStartsWith(path, depot):
2360                path = path[len(depot):]
2361
2362        else:
2363            for p in prefixes:
2364                if p4PathStartsWith(path, p):
2365                    path = path[len(p):]
2366                    break
2367
2368        path = wildcard_decode(path)
2369        return path
2370
2371    def splitFilesIntoBranches(self, commit):
2372        """Look at each depotFile in the commit to figure out to what
2373           branch it belongs."""
2374
2375        if self.clientSpecDirs:
2376            files = self.extractFilesFromCommit(commit)
2377            self.clientSpecDirs.update_client_spec_path_cache(files)
2378
2379        branches = {}
2380        fnum = 0
2381        while commit.has_key("depotFile%s" % fnum):
2382            path =  commit["depotFile%s" % fnum]
2383            found = [p for p in self.depotPaths
2384                     if p4PathStartsWith(path, p)]
2385            if not found:
2386                fnum = fnum + 1
2387                continue
2388
2389            file = {}
2390            file["path"] = path
2391            file["rev"] = commit["rev%s" % fnum]
2392            file["action"] = commit["action%s" % fnum]
2393            file["type"] = commit["type%s" % fnum]
2394            fnum = fnum + 1
2395
2396            # start with the full relative path where this file would
2397            # go in a p4 client
2398            if self.useClientSpec:
2399                relPath = self.clientSpecDirs.map_in_client(path)
2400            else:
2401                relPath = self.stripRepoPath(path, self.depotPaths)
2402
2403            for branch in self.knownBranches.keys():
2404                # add a trailing slash so that a commit into qt/4.2foo
2405                # doesn't end up in qt/4.2, e.g.
2406                if relPath.startswith(branch + "/"):
2407                    if branch not in branches:
2408                        branches[branch] = []
2409                    branches[branch].append(file)
2410                    break
2411
2412        return branches
2413
2414    def writeToGitStream(self, gitMode, relPath, contents):
2415        self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2416        self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2417        for d in contents:
2418            self.gitStream.write(d)
2419        self.gitStream.write('\n')
2420
2421    # output one file from the P4 stream
2422    # - helper for streamP4Files
2423
2424    def streamOneP4File(self, file, contents):
2425        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2426        if verbose:
2427            size = int(self.stream_file['fileSize'])
2428            sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2429            sys.stdout.flush()
2430
2431        (type_base, type_mods) = split_p4_type(file["type"])
2432
2433        git_mode = "100644"
2434        if "x" in type_mods:
2435            git_mode = "100755"
2436        if type_base == "symlink":
2437            git_mode = "120000"
2438            # p4 print on a symlink sometimes contains "target\n";
2439            # if it does, remove the newline
2440            data = ''.join(contents)
2441            if not data:
2442                # Some version of p4 allowed creating a symlink that pointed
2443                # to nothing.  This causes p4 errors when checking out such
2444                # a change, and errors here too.  Work around it by ignoring
2445                # the bad symlink; hopefully a future change fixes it.
2446                print "\nIgnoring empty symlink in %s" % file['depotFile']
2447                return
2448            elif data[-1] == '\n':
2449                contents = [data[:-1]]
2450            else:
2451                contents = [data]
2452
2453        if type_base == "utf16":
2454            # p4 delivers different text in the python output to -G
2455            # than it does when using "print -o", or normal p4 client
2456            # operations.  utf16 is converted to ascii or utf8, perhaps.
2457            # But ascii text saved as -t utf16 is completely mangled.
2458            # Invoke print -o to get the real contents.
2459            #
2460            # On windows, the newlines will always be mangled by print, so put
2461            # them back too.  This is not needed to the cygwin windows version,
2462            # just the native "NT" type.
2463            #
2464            try:
2465                text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2466            except Exception as e:
2467                if 'Translation of file content failed' in str(e):
2468                    type_base = 'binary'
2469                else:
2470                    raise e
2471            else:
2472                if p4_version_string().find('/NT') >= 0:
2473                    text = text.replace('\r\n', '\n')
2474                contents = [ text ]
2475
2476        if type_base == "apple":
2477            # Apple filetype files will be streamed as a concatenation of
2478            # its appledouble header and the contents.  This is useless
2479            # on both macs and non-macs.  If using "print -q -o xx", it
2480            # will create "xx" with the data, and "%xx" with the header.
2481            # This is also not very useful.
2482            #
2483            # Ideally, someday, this script can learn how to generate
2484            # appledouble files directly and import those to git, but
2485            # non-mac machines can never find a use for apple filetype.
2486            print "\nIgnoring apple filetype file %s" % file['depotFile']
2487            return
2488
2489        # Note that we do not try to de-mangle keywords on utf16 files,
2490        # even though in theory somebody may want that.
2491        pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2492        if pattern:
2493            regexp = re.compile(pattern, re.VERBOSE)
2494            text = ''.join(contents)
2495            text = regexp.sub(r'$\1$', text)
2496            contents = [ text ]
2497
2498        try:
2499            relPath.decode('ascii')
2500        except:
2501            encoding = 'utf8'
2502            if gitConfig('git-p4.pathEncoding'):
2503                encoding = gitConfig('git-p4.pathEncoding')
2504            relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2505            if self.verbose:
2506                print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2507
2508        if self.largeFileSystem:
2509            (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2510
2511        self.writeToGitStream(git_mode, relPath, contents)
2512
2513    def streamOneP4Deletion(self, file):
2514        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2515        if verbose:
2516            sys.stdout.write("delete %s\n" % relPath)
2517            sys.stdout.flush()
2518        self.gitStream.write("D %s\n" % relPath)
2519
2520        if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2521            self.largeFileSystem.removeLargeFile(relPath)
2522
2523    # handle another chunk of streaming data
2524    def streamP4FilesCb(self, marshalled):
2525
2526        # catch p4 errors and complain
2527        err = None
2528        if "code" in marshalled:
2529            if marshalled["code"] == "error":
2530                if "data" in marshalled:
2531                    err = marshalled["data"].rstrip()
2532
2533        if not err and 'fileSize' in self.stream_file:
2534            required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2535            if required_bytes > 0:
2536                err = 'Not enough space left on %s! Free at least %i MB.' % (
2537                    os.getcwd(), required_bytes/1024/1024
2538                )
2539
2540        if err:
2541            f = None
2542            if self.stream_have_file_info:
2543                if "depotFile" in self.stream_file:
2544                    f = self.stream_file["depotFile"]
2545            # force a failure in fast-import, else an empty
2546            # commit will be made
2547            self.gitStream.write("\n")
2548            self.gitStream.write("die-now\n")
2549            self.gitStream.close()
2550            # ignore errors, but make sure it exits first
2551            self.importProcess.wait()
2552            if f:
2553                die("Error from p4 print for %s: %s" % (f, err))
2554            else:
2555                die("Error from p4 print: %s" % err)
2556
2557        if marshalled.has_key('depotFile') and self.stream_have_file_info:
2558            # start of a new file - output the old one first
2559            self.streamOneP4File(self.stream_file, self.stream_contents)
2560            self.stream_file = {}
2561            self.stream_contents = []
2562            self.stream_have_file_info = False
2563
2564        # pick up the new file information... for the
2565        # 'data' field we need to append to our array
2566        for k in marshalled.keys():
2567            if k == 'data':
2568                if 'streamContentSize' not in self.stream_file:
2569                    self.stream_file['streamContentSize'] = 0
2570                self.stream_file['streamContentSize'] += len(marshalled['data'])
2571                self.stream_contents.append(marshalled['data'])
2572            else:
2573                self.stream_file[k] = marshalled[k]
2574
2575        if (verbose and
2576            'streamContentSize' in self.stream_file and
2577            'fileSize' in self.stream_file and
2578            'depotFile' in self.stream_file):
2579            size = int(self.stream_file["fileSize"])
2580            if size > 0:
2581                progress = 100*self.stream_file['streamContentSize']/size
2582                sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2583                sys.stdout.flush()
2584
2585        self.stream_have_file_info = True
2586
2587    # Stream directly from "p4 files" into "git fast-import"
2588    def streamP4Files(self, files):
2589        filesForCommit = []
2590        filesToRead = []
2591        filesToDelete = []
2592
2593        for f in files:
2594            filesForCommit.append(f)
2595            if f['action'] in self.delete_actions:
2596                filesToDelete.append(f)
2597            else:
2598                filesToRead.append(f)
2599
2600        # deleted files...
2601        for f in filesToDelete:
2602            self.streamOneP4Deletion(f)
2603
2604        if len(filesToRead) > 0:
2605            self.stream_file = {}
2606            self.stream_contents = []
2607            self.stream_have_file_info = False
2608
2609            # curry self argument
2610            def streamP4FilesCbSelf(entry):
2611                self.streamP4FilesCb(entry)
2612
2613            fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2614
2615            p4CmdList(["-x", "-", "print"],
2616                      stdin=fileArgs,
2617                      cb=streamP4FilesCbSelf)
2618
2619            # do the last chunk
2620            if self.stream_file.has_key('depotFile'):
2621                self.streamOneP4File(self.stream_file, self.stream_contents)
2622
2623    def make_email(self, userid):
2624        if userid in self.users:
2625            return self.users[userid]
2626        else:
2627            return "%s <a@b>" % userid
2628
2629    def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2630        """ Stream a p4 tag.
2631        commit is either a git commit, or a fast-import mark, ":<p4commit>"
2632        """
2633
2634        if verbose:
2635            print "writing tag %s for commit %s" % (labelName, commit)
2636        gitStream.write("tag %s\n" % labelName)
2637        gitStream.write("from %s\n" % commit)
2638
2639        if labelDetails.has_key('Owner'):
2640            owner = labelDetails["Owner"]
2641        else:
2642            owner = None
2643
2644        # Try to use the owner of the p4 label, or failing that,
2645        # the current p4 user id.
2646        if owner:
2647            email = self.make_email(owner)
2648        else:
2649            email = self.make_email(self.p4UserId())
2650        tagger = "%s %s %s" % (email, epoch, self.tz)
2651
2652        gitStream.write("tagger %s\n" % tagger)
2653
2654        print "labelDetails=",labelDetails
2655        if labelDetails.has_key('Description'):
2656            description = labelDetails['Description']
2657        else:
2658            description = 'Label from git p4'
2659
2660        gitStream.write("data %d\n" % len(description))
2661        gitStream.write(description)
2662        gitStream.write("\n")
2663
2664    def inClientSpec(self, path):
2665        if not self.clientSpecDirs:
2666            return True
2667        inClientSpec = self.clientSpecDirs.map_in_client(path)
2668        if not inClientSpec and self.verbose:
2669            print('Ignoring file outside of client spec: {0}'.format(path))
2670        return inClientSpec
2671
2672    def hasBranchPrefix(self, path):
2673        if not self.branchPrefixes:
2674            return True
2675        hasPrefix = [p for p in self.branchPrefixes
2676                        if p4PathStartsWith(path, p)]
2677        if not hasPrefix and self.verbose:
2678            print('Ignoring file outside of prefix: {0}'.format(path))
2679        return hasPrefix
2680
2681    def commit(self, details, files, branch, parent = ""):
2682        epoch = details["time"]
2683        author = details["user"]
2684        jobs = self.extractJobsFromCommit(details)
2685
2686        if self.verbose:
2687            print('commit into {0}'.format(branch))
2688
2689        if self.clientSpecDirs:
2690            self.clientSpecDirs.update_client_spec_path_cache(files)
2691
2692        files = [f for f in files
2693            if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2694
2695        if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2696            print('Ignoring revision {0} as it would produce an empty commit.'
2697                .format(details['change']))
2698            return
2699
2700        self.gitStream.write("commit %s\n" % branch)
2701        self.gitStream.write("mark :%s\n" % details["change"])
2702        self.committedChanges.add(int(details["change"]))
2703        committer = ""
2704        if author not in self.users:
2705            self.getUserMapFromPerforceServer()
2706        committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2707
2708        self.gitStream.write("committer %s\n" % committer)
2709
2710        self.gitStream.write("data <<EOT\n")
2711        self.gitStream.write(details["desc"])
2712        if len(jobs) > 0:
2713            self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2714        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2715                             (','.join(self.branchPrefixes), details["change"]))
2716        if len(details['options']) > 0:
2717            self.gitStream.write(": options = %s" % details['options'])
2718        self.gitStream.write("]\nEOT\n\n")
2719
2720        if len(parent) > 0:
2721            if self.verbose:
2722                print "parent %s" % parent
2723            self.gitStream.write("from %s\n" % parent)
2724
2725        self.streamP4Files(files)
2726        self.gitStream.write("\n")
2727
2728        change = int(details["change"])
2729
2730        if self.labels.has_key(change):
2731            label = self.labels[change]
2732            labelDetails = label[0]
2733            labelRevisions = label[1]
2734            if self.verbose:
2735                print "Change %s is labelled %s" % (change, labelDetails)
2736
2737            files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2738                                                for p in self.branchPrefixes])
2739
2740            if len(files) == len(labelRevisions):
2741
2742                cleanedFiles = {}
2743                for info in files:
2744                    if info["action"] in self.delete_actions:
2745                        continue
2746                    cleanedFiles[info["depotFile"]] = info["rev"]
2747
2748                if cleanedFiles == labelRevisions:
2749                    self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2750
2751                else:
2752                    if not self.silent:
2753                        print ("Tag %s does not match with change %s: files do not match."
2754                               % (labelDetails["label"], change))
2755
2756            else:
2757                if not self.silent:
2758                    print ("Tag %s does not match with change %s: file count is different."
2759                           % (labelDetails["label"], change))
2760
2761    # Build a dictionary of changelists and labels, for "detect-labels" option.
2762    def getLabels(self):
2763        self.labels = {}
2764
2765        l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2766        if len(l) > 0 and not self.silent:
2767            print "Finding files belonging to labels in %s" % `self.depotPaths`
2768
2769        for output in l:
2770            label = output["label"]
2771            revisions = {}
2772            newestChange = 0
2773            if self.verbose:
2774                print "Querying files for label %s" % label
2775            for file in p4CmdList(["files"] +
2776                                      ["%s...@%s" % (p, label)
2777                                          for p in self.depotPaths]):
2778                revisions[file["depotFile"]] = file["rev"]
2779                change = int(file["change"])
2780                if change > newestChange:
2781                    newestChange = change
2782
2783            self.labels[newestChange] = [output, revisions]
2784
2785        if self.verbose:
2786            print "Label changes: %s" % self.labels.keys()
2787
2788    # Import p4 labels as git tags. A direct mapping does not
2789    # exist, so assume that if all the files are at the same revision
2790    # then we can use that, or it's something more complicated we should
2791    # just ignore.
2792    def importP4Labels(self, stream, p4Labels):
2793        if verbose:
2794            print "import p4 labels: " + ' '.join(p4Labels)
2795
2796        ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2797        validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2798        if len(validLabelRegexp) == 0:
2799            validLabelRegexp = defaultLabelRegexp
2800        m = re.compile(validLabelRegexp)
2801
2802        for name in p4Labels:
2803            commitFound = False
2804
2805            if not m.match(name):
2806                if verbose:
2807                    print "label %s does not match regexp %s" % (name,validLabelRegexp)
2808                continue
2809
2810            if name in ignoredP4Labels:
2811                continue
2812
2813            labelDetails = p4CmdList(['label', "-o", name])[0]
2814
2815            # get the most recent changelist for each file in this label
2816            change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2817                                for p in self.depotPaths])
2818
2819            if change.has_key('change'):
2820                # find the corresponding git commit; take the oldest commit
2821                changelist = int(change['change'])
2822                if changelist in self.committedChanges:
2823                    gitCommit = ":%d" % changelist       # use a fast-import mark
2824                    commitFound = True
2825                else:
2826                    gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2827                        "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2828                    if len(gitCommit) == 0:
2829                        print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2830                    else:
2831                        commitFound = True
2832                        gitCommit = gitCommit.strip()
2833
2834                if commitFound:
2835                    # Convert from p4 time format
2836                    try:
2837                        tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2838                    except ValueError:
2839                        print "Could not convert label time %s" % labelDetails['Update']
2840                        tmwhen = 1
2841
2842                    when = int(time.mktime(tmwhen))
2843                    self.streamTag(stream, name, labelDetails, gitCommit, when)
2844                    if verbose:
2845                        print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2846            else:
2847                if verbose:
2848                    print "Label %s has no changelists - possibly deleted?" % name
2849
2850            if not commitFound:
2851                # We can't import this label; don't try again as it will get very
2852                # expensive repeatedly fetching all the files for labels that will
2853                # never be imported. If the label is moved in the future, the
2854                # ignore will need to be removed manually.
2855                system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2856
2857    def guessProjectName(self):
2858        for p in self.depotPaths:
2859            if p.endswith("/"):
2860                p = p[:-1]
2861            p = p[p.strip().rfind("/") + 1:]
2862            if not p.endswith("/"):
2863               p += "/"
2864            return p
2865
2866    def getBranchMapping(self):
2867        lostAndFoundBranches = set()
2868
2869        user = gitConfig("git-p4.branchUser")
2870        if len(user) > 0:
2871            command = "branches -u %s" % user
2872        else:
2873            command = "branches"
2874
2875        for info in p4CmdList(command):
2876            details = p4Cmd(["branch", "-o", info["branch"]])
2877            viewIdx = 0
2878            while details.has_key("View%s" % viewIdx):
2879                paths = details["View%s" % viewIdx].split(" ")
2880                viewIdx = viewIdx + 1
2881                # require standard //depot/foo/... //depot/bar/... mapping
2882                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2883                    continue
2884                source = paths[0]
2885                destination = paths[1]
2886                ## HACK
2887                if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2888                    source = source[len(self.depotPaths[0]):-4]
2889                    destination = destination[len(self.depotPaths[0]):-4]
2890
2891                    if destination in self.knownBranches:
2892                        if not self.silent:
2893                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2894                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2895                        continue
2896
2897                    self.knownBranches[destination] = source
2898
2899                    lostAndFoundBranches.discard(destination)
2900
2901                    if source not in self.knownBranches:
2902                        lostAndFoundBranches.add(source)
2903
2904        # Perforce does not strictly require branches to be defined, so we also
2905        # check git config for a branch list.
2906        #
2907        # Example of branch definition in git config file:
2908        # [git-p4]
2909        #   branchList=main:branchA
2910        #   branchList=main:branchB
2911        #   branchList=branchA:branchC
2912        configBranches = gitConfigList("git-p4.branchList")
2913        for branch in configBranches:
2914            if branch:
2915                (source, destination) = branch.split(":")
2916                self.knownBranches[destination] = source
2917
2918                lostAndFoundBranches.discard(destination)
2919
2920                if source not in self.knownBranches:
2921                    lostAndFoundBranches.add(source)
2922
2923
2924        for branch in lostAndFoundBranches:
2925            self.knownBranches[branch] = branch
2926
2927    def getBranchMappingFromGitBranches(self):
2928        branches = p4BranchesInGit(self.importIntoRemotes)
2929        for branch in branches.keys():
2930            if branch == "master":
2931                branch = "main"
2932            else:
2933                branch = branch[len(self.projectName):]
2934            self.knownBranches[branch] = branch
2935
2936    def updateOptionDict(self, d):
2937        option_keys = {}
2938        if self.keepRepoPath:
2939            option_keys['keepRepoPath'] = 1
2940
2941        d["options"] = ' '.join(sorted(option_keys.keys()))
2942
2943    def readOptions(self, d):
2944        self.keepRepoPath = (d.has_key('options')
2945                             and ('keepRepoPath' in d['options']))
2946
2947    def gitRefForBranch(self, branch):
2948        if branch == "main":
2949            return self.refPrefix + "master"
2950
2951        if len(branch) <= 0:
2952            return branch
2953
2954        return self.refPrefix + self.projectName + branch
2955
2956    def gitCommitByP4Change(self, ref, change):
2957        if self.verbose:
2958            print "looking in ref " + ref + " for change %s using bisect..." % change
2959
2960        earliestCommit = ""
2961        latestCommit = parseRevision(ref)
2962
2963        while True:
2964            if self.verbose:
2965                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2966            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2967            if len(next) == 0:
2968                if self.verbose:
2969                    print "argh"
2970                return ""
2971            log = extractLogMessageFromGitCommit(next)
2972            settings = extractSettingsGitLog(log)
2973            currentChange = int(settings['change'])
2974            if self.verbose:
2975                print "current change %s" % currentChange
2976
2977            if currentChange == change:
2978                if self.verbose:
2979                    print "found %s" % next
2980                return next
2981
2982            if currentChange < change:
2983                earliestCommit = "^%s" % next
2984            else:
2985                latestCommit = "%s" % next
2986
2987        return ""
2988
2989    def importNewBranch(self, branch, maxChange):
2990        # make fast-import flush all changes to disk and update the refs using the checkpoint
2991        # command so that we can try to find the branch parent in the git history
2992        self.gitStream.write("checkpoint\n\n");
2993        self.gitStream.flush();
2994        branchPrefix = self.depotPaths[0] + branch + "/"
2995        range = "@1,%s" % maxChange
2996        #print "prefix" + branchPrefix
2997        changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
2998        if len(changes) <= 0:
2999            return False
3000        firstChange = changes[0]
3001        #print "first change in branch: %s" % firstChange
3002        sourceBranch = self.knownBranches[branch]
3003        sourceDepotPath = self.depotPaths[0] + sourceBranch
3004        sourceRef = self.gitRefForBranch(sourceBranch)
3005        #print "source " + sourceBranch
3006
3007        branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3008        #print "branch parent: %s" % branchParentChange
3009        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3010        if len(gitParent) > 0:
3011            self.initialParents[self.gitRefForBranch(branch)] = gitParent
3012            #print "parent git commit: %s" % gitParent
3013
3014        self.importChanges(changes)
3015        return True
3016
3017    def searchParent(self, parent, branch, target):
3018        parentFound = False
3019        for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3020                                     "--no-merges", parent]):
3021            blob = blob.strip()
3022            if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3023                parentFound = True
3024                if self.verbose:
3025                    print "Found parent of %s in commit %s" % (branch, blob)
3026                break
3027        if parentFound:
3028            return blob
3029        else:
3030            return None
3031
3032    def importChanges(self, changes):
3033        cnt = 1
3034        for change in changes:
3035            description = p4_describe(change)
3036            self.updateOptionDict(description)
3037
3038            if not self.silent:
3039                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3040                sys.stdout.flush()
3041            cnt = cnt + 1
3042
3043            try:
3044                if self.detectBranches:
3045                    branches = self.splitFilesIntoBranches(description)
3046                    for branch in branches.keys():
3047                        ## HACK  --hwn
3048                        branchPrefix = self.depotPaths[0] + branch + "/"
3049                        self.branchPrefixes = [ branchPrefix ]
3050
3051                        parent = ""
3052
3053                        filesForCommit = branches[branch]
3054
3055                        if self.verbose:
3056                            print "branch is %s" % branch
3057
3058                        self.updatedBranches.add(branch)
3059
3060                        if branch not in self.createdBranches:
3061                            self.createdBranches.add(branch)
3062                            parent = self.knownBranches[branch]
3063                            if parent == branch:
3064                                parent = ""
3065                            else:
3066                                fullBranch = self.projectName + branch
3067                                if fullBranch not in self.p4BranchesInGit:
3068                                    if not self.silent:
3069                                        print("\n    Importing new branch %s" % fullBranch);
3070                                    if self.importNewBranch(branch, change - 1):
3071                                        parent = ""
3072                                        self.p4BranchesInGit.append(fullBranch)
3073                                    if not self.silent:
3074                                        print("\n    Resuming with change %s" % change);
3075
3076                                if self.verbose:
3077                                    print "parent determined through known branches: %s" % parent
3078
3079                        branch = self.gitRefForBranch(branch)
3080                        parent = self.gitRefForBranch(parent)
3081
3082                        if self.verbose:
3083                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3084
3085                        if len(parent) == 0 and branch in self.initialParents:
3086                            parent = self.initialParents[branch]
3087                            del self.initialParents[branch]
3088
3089                        blob = None
3090                        if len(parent) > 0:
3091                            tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3092                            if self.verbose:
3093                                print "Creating temporary branch: " + tempBranch
3094                            self.commit(description, filesForCommit, tempBranch)
3095                            self.tempBranches.append(tempBranch)
3096                            self.checkpoint()
3097                            blob = self.searchParent(parent, branch, tempBranch)
3098                        if blob:
3099                            self.commit(description, filesForCommit, branch, blob)
3100                        else:
3101                            if self.verbose:
3102                                print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3103                            self.commit(description, filesForCommit, branch, parent)
3104                else:
3105                    files = self.extractFilesFromCommit(description)
3106                    self.commit(description, files, self.branch,
3107                                self.initialParent)
3108                    # only needed once, to connect to the previous commit
3109                    self.initialParent = ""
3110            except IOError:
3111                print self.gitError.read()
3112                sys.exit(1)
3113
3114    def importHeadRevision(self, revision):
3115        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3116
3117        details = {}
3118        details["user"] = "git perforce import user"
3119        details["desc"] = ("Initial import of %s from the state at revision %s\n"
3120                           % (' '.join(self.depotPaths), revision))
3121        details["change"] = revision
3122        newestRevision = 0
3123
3124        fileCnt = 0
3125        fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3126
3127        for info in p4CmdList(["files"] + fileArgs):
3128
3129            if 'code' in info and info['code'] == 'error':
3130                sys.stderr.write("p4 returned an error: %s\n"
3131                                 % info['data'])
3132                if info['data'].find("must refer to client") >= 0:
3133                    sys.stderr.write("This particular p4 error is misleading.\n")
3134                    sys.stderr.write("Perhaps the depot path was misspelled.\n");
3135                    sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3136                sys.exit(1)
3137            if 'p4ExitCode' in info:
3138                sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3139                sys.exit(1)
3140
3141
3142            change = int(info["change"])
3143            if change > newestRevision:
3144                newestRevision = change
3145
3146            if info["action"] in self.delete_actions:
3147                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3148                #fileCnt = fileCnt + 1
3149                continue
3150
3151            for prop in ["depotFile", "rev", "action", "type" ]:
3152                details["%s%s" % (prop, fileCnt)] = info[prop]
3153
3154            fileCnt = fileCnt + 1
3155
3156        details["change"] = newestRevision
3157
3158        # Use time from top-most change so that all git p4 clones of
3159        # the same p4 repo have the same commit SHA1s.
3160        res = p4_describe(newestRevision)
3161        details["time"] = res["time"]
3162
3163        self.updateOptionDict(details)
3164        try:
3165            self.commit(details, self.extractFilesFromCommit(details), self.branch)
3166        except IOError:
3167            print "IO error with git fast-import. Is your git version recent enough?"
3168            print self.gitError.read()
3169
3170
3171    def run(self, args):
3172        self.depotPaths = []
3173        self.changeRange = ""
3174        self.previousDepotPaths = []
3175        self.hasOrigin = False
3176
3177        # map from branch depot path to parent branch
3178        self.knownBranches = {}
3179        self.initialParents = {}
3180
3181        if self.importIntoRemotes:
3182            self.refPrefix = "refs/remotes/p4/"
3183        else:
3184            self.refPrefix = "refs/heads/p4/"
3185
3186        if self.syncWithOrigin:
3187            self.hasOrigin = originP4BranchesExist()
3188            if self.hasOrigin:
3189                if not self.silent:
3190                    print 'Syncing with origin first, using "git fetch origin"'
3191                system("git fetch origin")
3192
3193        branch_arg_given = bool(self.branch)
3194        if len(self.branch) == 0:
3195            self.branch = self.refPrefix + "master"
3196            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3197                system("git update-ref %s refs/heads/p4" % self.branch)
3198                system("git branch -D p4")
3199
3200        # accept either the command-line option, or the configuration variable
3201        if self.useClientSpec:
3202            # will use this after clone to set the variable
3203            self.useClientSpec_from_options = True
3204        else:
3205            if gitConfigBool("git-p4.useclientspec"):
3206                self.useClientSpec = True
3207        if self.useClientSpec:
3208            self.clientSpecDirs = getClientSpec()
3209
3210        # TODO: should always look at previous commits,
3211        # merge with previous imports, if possible.
3212        if args == []:
3213            if self.hasOrigin:
3214                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3215
3216            # branches holds mapping from branch name to sha1
3217            branches = p4BranchesInGit(self.importIntoRemotes)
3218
3219            # restrict to just this one, disabling detect-branches
3220            if branch_arg_given:
3221                short = self.branch.split("/")[-1]
3222                if short in branches:
3223                    self.p4BranchesInGit = [ short ]
3224            else:
3225                self.p4BranchesInGit = branches.keys()
3226
3227            if len(self.p4BranchesInGit) > 1:
3228                if not self.silent:
3229                    print "Importing from/into multiple branches"
3230                self.detectBranches = True
3231                for branch in branches.keys():
3232                    self.initialParents[self.refPrefix + branch] = \
3233                        branches[branch]
3234
3235            if self.verbose:
3236                print "branches: %s" % self.p4BranchesInGit
3237
3238            p4Change = 0
3239            for branch in self.p4BranchesInGit:
3240                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3241
3242                settings = extractSettingsGitLog(logMsg)
3243
3244                self.readOptions(settings)
3245                if (settings.has_key('depot-paths')
3246                    and settings.has_key ('change')):
3247                    change = int(settings['change']) + 1
3248                    p4Change = max(p4Change, change)
3249
3250                    depotPaths = sorted(settings['depot-paths'])
3251                    if self.previousDepotPaths == []:
3252                        self.previousDepotPaths = depotPaths
3253                    else:
3254                        paths = []
3255                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3256                            prev_list = prev.split("/")
3257                            cur_list = cur.split("/")
3258                            for i in range(0, min(len(cur_list), len(prev_list))):
3259                                if cur_list[i] <> prev_list[i]:
3260                                    i = i - 1
3261                                    break
3262
3263                            paths.append ("/".join(cur_list[:i + 1]))
3264
3265                        self.previousDepotPaths = paths
3266
3267            if p4Change > 0:
3268                self.depotPaths = sorted(self.previousDepotPaths)
3269                self.changeRange = "@%s,#head" % p4Change
3270                if not self.silent and not self.detectBranches:
3271                    print "Performing incremental import into %s git branch" % self.branch
3272
3273        # accept multiple ref name abbreviations:
3274        #    refs/foo/bar/branch -> use it exactly
3275        #    p4/branch -> prepend refs/remotes/ or refs/heads/
3276        #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3277        if not self.branch.startswith("refs/"):
3278            if self.importIntoRemotes:
3279                prepend = "refs/remotes/"
3280            else:
3281                prepend = "refs/heads/"
3282            if not self.branch.startswith("p4/"):
3283                prepend += "p4/"
3284            self.branch = prepend + self.branch
3285
3286        if len(args) == 0 and self.depotPaths:
3287            if not self.silent:
3288                print "Depot paths: %s" % ' '.join(self.depotPaths)
3289        else:
3290            if self.depotPaths and self.depotPaths != args:
3291                print ("previous import used depot path %s and now %s was specified. "
3292                       "This doesn't work!" % (' '.join (self.depotPaths),
3293                                               ' '.join (args)))
3294                sys.exit(1)
3295
3296            self.depotPaths = sorted(args)
3297
3298        revision = ""
3299        self.users = {}
3300
3301        # Make sure no revision specifiers are used when --changesfile
3302        # is specified.
3303        bad_changesfile = False
3304        if len(self.changesFile) > 0:
3305            for p in self.depotPaths:
3306                if p.find("@") >= 0 or p.find("#") >= 0:
3307                    bad_changesfile = True
3308                    break
3309        if bad_changesfile:
3310            die("Option --changesfile is incompatible with revision specifiers")
3311
3312        newPaths = []
3313        for p in self.depotPaths:
3314            if p.find("@") != -1:
3315                atIdx = p.index("@")
3316                self.changeRange = p[atIdx:]
3317                if self.changeRange == "@all":
3318                    self.changeRange = ""
3319                elif ',' not in self.changeRange:
3320                    revision = self.changeRange
3321                    self.changeRange = ""
3322                p = p[:atIdx]
3323            elif p.find("#") != -1:
3324                hashIdx = p.index("#")
3325                revision = p[hashIdx:]
3326                p = p[:hashIdx]
3327            elif self.previousDepotPaths == []:
3328                # pay attention to changesfile, if given, else import
3329                # the entire p4 tree at the head revision
3330                if len(self.changesFile) == 0:
3331                    revision = "#head"
3332
3333            p = re.sub ("\.\.\.$", "", p)
3334            if not p.endswith("/"):
3335                p += "/"
3336
3337            newPaths.append(p)
3338
3339        self.depotPaths = newPaths
3340
3341        # --detect-branches may change this for each branch
3342        self.branchPrefixes = self.depotPaths
3343
3344        self.loadUserMapFromCache()
3345        self.labels = {}
3346        if self.detectLabels:
3347            self.getLabels();
3348
3349        if self.detectBranches:
3350            ## FIXME - what's a P4 projectName ?
3351            self.projectName = self.guessProjectName()
3352
3353            if self.hasOrigin:
3354                self.getBranchMappingFromGitBranches()
3355            else:
3356                self.getBranchMapping()
3357            if self.verbose:
3358                print "p4-git branches: %s" % self.p4BranchesInGit
3359                print "initial parents: %s" % self.initialParents
3360            for b in self.p4BranchesInGit:
3361                if b != "master":
3362
3363                    ## FIXME
3364                    b = b[len(self.projectName):]
3365                self.createdBranches.add(b)
3366
3367        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3368
3369        self.importProcess = subprocess.Popen(["git", "fast-import"],
3370                                              stdin=subprocess.PIPE,
3371                                              stdout=subprocess.PIPE,
3372                                              stderr=subprocess.PIPE);
3373        self.gitOutput = self.importProcess.stdout
3374        self.gitStream = self.importProcess.stdin
3375        self.gitError = self.importProcess.stderr
3376
3377        if revision:
3378            self.importHeadRevision(revision)
3379        else:
3380            changes = []
3381
3382            if len(self.changesFile) > 0:
3383                output = open(self.changesFile).readlines()
3384                changeSet = set()
3385                for line in output:
3386                    changeSet.add(int(line))
3387
3388                for change in changeSet:
3389                    changes.append(change)
3390
3391                changes.sort()
3392            else:
3393                # catch "git p4 sync" with no new branches, in a repo that
3394                # does not have any existing p4 branches
3395                if len(args) == 0:
3396                    if not self.p4BranchesInGit:
3397                        die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3398
3399                    # The default branch is master, unless --branch is used to
3400                    # specify something else.  Make sure it exists, or complain
3401                    # nicely about how to use --branch.
3402                    if not self.detectBranches:
3403                        if not branch_exists(self.branch):
3404                            if branch_arg_given:
3405                                die("Error: branch %s does not exist." % self.branch)
3406                            else:
3407                                die("Error: no branch %s; perhaps specify one with --branch." %
3408                                    self.branch)
3409
3410                if self.verbose:
3411                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3412                                                              self.changeRange)
3413                changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3414
3415                if len(self.maxChanges) > 0:
3416                    changes = changes[:min(int(self.maxChanges), len(changes))]
3417
3418            if len(changes) == 0:
3419                if not self.silent:
3420                    print "No changes to import!"
3421            else:
3422                if not self.silent and not self.detectBranches:
3423                    print "Import destination: %s" % self.branch
3424
3425                self.updatedBranches = set()
3426
3427                if not self.detectBranches:
3428                    if args:
3429                        # start a new branch
3430                        self.initialParent = ""
3431                    else:
3432                        # build on a previous revision
3433                        self.initialParent = parseRevision(self.branch)
3434
3435                self.importChanges(changes)
3436
3437                if not self.silent:
3438                    print ""
3439                    if len(self.updatedBranches) > 0:
3440                        sys.stdout.write("Updated branches: ")
3441                        for b in self.updatedBranches:
3442                            sys.stdout.write("%s " % b)
3443                        sys.stdout.write("\n")
3444
3445        if gitConfigBool("git-p4.importLabels"):
3446            self.importLabels = True
3447
3448        if self.importLabels:
3449            p4Labels = getP4Labels(self.depotPaths)
3450            gitTags = getGitTags()
3451
3452            missingP4Labels = p4Labels - gitTags
3453            self.importP4Labels(self.gitStream, missingP4Labels)
3454
3455        self.gitStream.close()
3456        if self.importProcess.wait() != 0:
3457            die("fast-import failed: %s" % self.gitError.read())
3458        self.gitOutput.close()
3459        self.gitError.close()
3460
3461        # Cleanup temporary branches created during import
3462        if self.tempBranches != []:
3463            for branch in self.tempBranches:
3464                read_pipe("git update-ref -d %s" % branch)
3465            os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3466
3467        # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3468        # a convenient shortcut refname "p4".
3469        if self.importIntoRemotes:
3470            head_ref = self.refPrefix + "HEAD"
3471            if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3472                system(["git", "symbolic-ref", head_ref, self.branch])
3473
3474        return True
3475
3476class P4Rebase(Command):
3477    def __init__(self):
3478        Command.__init__(self)
3479        self.options = [
3480                optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3481        ]
3482        self.importLabels = False
3483        self.description = ("Fetches the latest revision from perforce and "
3484                            + "rebases the current work (branch) against it")
3485
3486    def run(self, args):
3487        sync = P4Sync()
3488        sync.importLabels = self.importLabels
3489        sync.run([])
3490
3491        return self.rebase()
3492
3493    def rebase(self):
3494        if os.system("git update-index --refresh") != 0:
3495            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3496        if len(read_pipe("git diff-index HEAD --")) > 0:
3497            die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3498
3499        [upstream, settings] = findUpstreamBranchPoint()
3500        if len(upstream) == 0:
3501            die("Cannot find upstream branchpoint for rebase")
3502
3503        # the branchpoint may be p4/foo~3, so strip off the parent
3504        upstream = re.sub("~[0-9]+$", "", upstream)
3505
3506        print "Rebasing the current branch onto %s" % upstream
3507        oldHead = read_pipe("git rev-parse HEAD").strip()
3508        system("git rebase %s" % upstream)
3509        system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3510        return True
3511
3512class P4Clone(P4Sync):
3513    def __init__(self):
3514        P4Sync.__init__(self)
3515        self.description = "Creates a new git repository and imports from Perforce into it"
3516        self.usage = "usage: %prog [options] //depot/path[@revRange]"
3517        self.options += [
3518            optparse.make_option("--destination", dest="cloneDestination",
3519                                 action='store', default=None,
3520                                 help="where to leave result of the clone"),
3521            optparse.make_option("--bare", dest="cloneBare",
3522                                 action="store_true", default=False),
3523        ]
3524        self.cloneDestination = None
3525        self.needsGit = False
3526        self.cloneBare = False
3527
3528    def defaultDestination(self, args):
3529        ## TODO: use common prefix of args?
3530        depotPath = args[0]
3531        depotDir = re.sub("(@[^@]*)$", "", depotPath)
3532        depotDir = re.sub("(#[^#]*)$", "", depotDir)
3533        depotDir = re.sub(r"\.\.\.$", "", depotDir)
3534        depotDir = re.sub(r"/$", "", depotDir)
3535        return os.path.split(depotDir)[1]
3536
3537    def run(self, args):
3538        if len(args) < 1:
3539            return False
3540
3541        if self.keepRepoPath and not self.cloneDestination:
3542            sys.stderr.write("Must specify destination for --keep-path\n")
3543            sys.exit(1)
3544
3545        depotPaths = args
3546
3547        if not self.cloneDestination and len(depotPaths) > 1:
3548            self.cloneDestination = depotPaths[-1]
3549            depotPaths = depotPaths[:-1]
3550
3551        self.cloneExclude = ["/"+p for p in self.cloneExclude]
3552        for p in depotPaths:
3553            if not p.startswith("//"):
3554                sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3555                return False
3556
3557        if not self.cloneDestination:
3558            self.cloneDestination = self.defaultDestination(args)
3559
3560        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3561
3562        if not os.path.exists(self.cloneDestination):
3563            os.makedirs(self.cloneDestination)
3564        chdir(self.cloneDestination)
3565
3566        init_cmd = [ "git", "init" ]
3567        if self.cloneBare:
3568            init_cmd.append("--bare")
3569        retcode = subprocess.call(init_cmd)
3570        if retcode:
3571            raise CalledProcessError(retcode, init_cmd)
3572
3573        if not P4Sync.run(self, depotPaths):
3574            return False
3575
3576        # create a master branch and check out a work tree
3577        if gitBranchExists(self.branch):
3578            system([ "git", "branch", "master", self.branch ])
3579            if not self.cloneBare:
3580                system([ "git", "checkout", "-f" ])
3581        else:
3582            print 'Not checking out any branch, use ' \
3583                  '"git checkout -q -b master <branch>"'
3584
3585        # auto-set this variable if invoked with --use-client-spec
3586        if self.useClientSpec_from_options:
3587            system("git config --bool git-p4.useclientspec true")
3588
3589        return True
3590
3591class P4Branches(Command):
3592    def __init__(self):
3593        Command.__init__(self)
3594        self.options = [ ]
3595        self.description = ("Shows the git branches that hold imports and their "
3596                            + "corresponding perforce depot paths")
3597        self.verbose = False
3598
3599    def run(self, args):
3600        if originP4BranchesExist():
3601            createOrUpdateBranchesFromOrigin()
3602
3603        cmdline = "git rev-parse --symbolic "
3604        cmdline += " --remotes"
3605
3606        for line in read_pipe_lines(cmdline):
3607            line = line.strip()
3608
3609            if not line.startswith('p4/') or line == "p4/HEAD":
3610                continue
3611            branch = line
3612
3613            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3614            settings = extractSettingsGitLog(log)
3615
3616            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3617        return True
3618
3619class HelpFormatter(optparse.IndentedHelpFormatter):
3620    def __init__(self):
3621        optparse.IndentedHelpFormatter.__init__(self)
3622
3623    def format_description(self, description):
3624        if description:
3625            return description + "\n"
3626        else:
3627            return ""
3628
3629def printUsage(commands):
3630    print "usage: %s <command> [options]" % sys.argv[0]
3631    print ""
3632    print "valid commands: %s" % ", ".join(commands)
3633    print ""
3634    print "Try %s <command> --help for command specific help." % sys.argv[0]
3635    print ""
3636
3637commands = {
3638    "debug" : P4Debug,
3639    "submit" : P4Submit,
3640    "commit" : P4Submit,
3641    "sync" : P4Sync,
3642    "rebase" : P4Rebase,
3643    "clone" : P4Clone,
3644    "rollback" : P4RollBack,
3645    "branches" : P4Branches
3646}
3647
3648
3649def main():
3650    if len(sys.argv[1:]) == 0:
3651        printUsage(commands.keys())
3652        sys.exit(2)
3653
3654    cmdName = sys.argv[1]
3655    try:
3656        klass = commands[cmdName]
3657        cmd = klass()
3658    except KeyError:
3659        print "unknown command %s" % cmdName
3660        print ""
3661        printUsage(commands.keys())
3662        sys.exit(2)
3663
3664    options = cmd.options
3665    cmd.gitdir = os.environ.get("GIT_DIR", None)
3666
3667    args = sys.argv[2:]
3668
3669    options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3670    if cmd.needsGit:
3671        options.append(optparse.make_option("--git-dir", dest="gitdir"))
3672
3673    parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3674                                   options,
3675                                   description = cmd.description,
3676                                   formatter = HelpFormatter())
3677
3678    (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3679    global verbose
3680    verbose = cmd.verbose
3681    if cmd.needsGit:
3682        if cmd.gitdir == None:
3683            cmd.gitdir = os.path.abspath(".git")
3684            if not isValidGitDir(cmd.gitdir):
3685                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3686                if os.path.exists(cmd.gitdir):
3687                    cdup = read_pipe("git rev-parse --show-cdup").strip()
3688                    if len(cdup) > 0:
3689                        chdir(cdup);
3690
3691        if not isValidGitDir(cmd.gitdir):
3692            if isValidGitDir(cmd.gitdir + "/.git"):
3693                cmd.gitdir += "/.git"
3694            else:
3695                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3696
3697        os.environ["GIT_DIR"] = cmd.gitdir
3698
3699    if not cmd.run(args):
3700        parser.print_help()
3701        sys.exit(2)
3702
3703
3704if __name__ == '__main__':
3705    main()