4425220bf62afc22f5a6e2c8882411b6d5781267
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
7# 2007 Trolltech ASA
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
10
11import optparse, sys, os, marshal, subprocess, shelve
12import tempfile, getopt, os.path, time, platform
13import re
14
15verbose = False
16
17
18def p4_build_cmd(cmd):
19 """Build a suitable p4 command line.
20
21 This consolidates building and returning a p4 command line into one
22 location. It means that hooking into the environment, or other configuration
23 can be done more easily.
24 """
25 real_cmd = "%s " % "p4"
26
27 user = gitConfig("git-p4.user")
28 if len(user) > 0:
29 real_cmd += "-u %s " % user
30
31 password = gitConfig("git-p4.password")
32 if len(password) > 0:
33 real_cmd += "-P %s " % password
34
35 port = gitConfig("git-p4.port")
36 if len(port) > 0:
37 real_cmd += "-p %s " % port
38
39 host = gitConfig("git-p4.host")
40 if len(host) > 0:
41 real_cmd += "-h %s " % host
42
43 client = gitConfig("git-p4.client")
44 if len(client) > 0:
45 real_cmd += "-c %s " % client
46
47 real_cmd += "%s" % (cmd)
48 if verbose:
49 print real_cmd
50 return real_cmd
51
52def chdir(dir):
53 if os.name == 'nt':
54 os.environ['PWD']=dir
55 os.chdir(dir)
56
57def die(msg):
58 if verbose:
59 raise Exception(msg)
60 else:
61 sys.stderr.write(msg + "\n")
62 sys.exit(1)
63
64def write_pipe(c, str):
65 if verbose:
66 sys.stderr.write('Writing pipe: %s\n' % c)
67
68 pipe = os.popen(c, 'w')
69 val = pipe.write(str)
70 if pipe.close():
71 die('Command failed: %s' % c)
72
73 return val
74
75def p4_write_pipe(c, str):
76 real_cmd = p4_build_cmd(c)
77 return write_pipe(real_cmd, str)
78
79def read_pipe(c, ignore_error=False):
80 if verbose:
81 sys.stderr.write('Reading pipe: %s\n' % c)
82
83 pipe = os.popen(c, 'rb')
84 val = pipe.read()
85 if pipe.close() and not ignore_error:
86 die('Command failed: %s' % c)
87
88 return val
89
90def p4_read_pipe(c, ignore_error=False):
91 real_cmd = p4_build_cmd(c)
92 return read_pipe(real_cmd, ignore_error)
93
94def read_pipe_lines(c):
95 if verbose:
96 sys.stderr.write('Reading pipe: %s\n' % c)
97 ## todo: check return status
98 pipe = os.popen(c, 'rb')
99 val = pipe.readlines()
100 if pipe.close():
101 die('Command failed: %s' % c)
102
103 return val
104
105def p4_read_pipe_lines(c):
106 """Specifically invoke p4 on the command supplied. """
107 real_cmd = p4_build_cmd(c)
108 return read_pipe_lines(real_cmd)
109
110def system(cmd):
111 if verbose:
112 sys.stderr.write("executing %s\n" % cmd)
113 if os.system(cmd) != 0:
114 die("command failed: %s" % cmd)
115
116def p4_system(cmd):
117 """Specifically invoke p4 as the system command. """
118 real_cmd = p4_build_cmd(cmd)
119 return system(real_cmd)
120
121def isP4Exec(kind):
122 """Determine if a Perforce 'kind' should have execute permission
123
124 'p4 help filetypes' gives a list of the types. If it starts with 'x',
125 or x follows one of a few letters. Otherwise, if there is an 'x' after
126 a plus sign, it is also executable"""
127 return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
128
129def setP4ExecBit(file, mode):
130 # Reopens an already open file and changes the execute bit to match
131 # the execute bit setting in the passed in mode.
132
133 p4Type = "+x"
134
135 if not isModeExec(mode):
136 p4Type = getP4OpenedType(file)
137 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
138 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
139 if p4Type[-1] == "+":
140 p4Type = p4Type[0:-1]
141
142 p4_system("reopen -t %s %s" % (p4Type, file))
143
144def getP4OpenedType(file):
145 # Returns the perforce file type for the given file.
146
147 result = p4_read_pipe("opened %s" % file)
148 match = re.match(".*\((.+)\)\r?$", result)
149 if match:
150 return match.group(1)
151 else:
152 die("Could not determine file type for %s (result: '%s')" % (file, result))
153
154def diffTreePattern():
155 # This is a simple generator for the diff tree regex pattern. This could be
156 # a class variable if this and parseDiffTreeEntry were a part of a class.
157 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
158 while True:
159 yield pattern
160
161def parseDiffTreeEntry(entry):
162 """Parses a single diff tree entry into its component elements.
163
164 See git-diff-tree(1) manpage for details about the format of the diff
165 output. This method returns a dictionary with the following elements:
166
167 src_mode - The mode of the source file
168 dst_mode - The mode of the destination file
169 src_sha1 - The sha1 for the source file
170 dst_sha1 - The sha1 fr the destination file
171 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
172 status_score - The score for the status (applicable for 'C' and 'R'
173 statuses). This is None if there is no score.
174 src - The path for the source file.
175 dst - The path for the destination file. This is only present for
176 copy or renames. If it is not present, this is None.
177
178 If the pattern is not matched, None is returned."""
179
180 match = diffTreePattern().next().match(entry)
181 if match:
182 return {
183 'src_mode': match.group(1),
184 'dst_mode': match.group(2),
185 'src_sha1': match.group(3),
186 'dst_sha1': match.group(4),
187 'status': match.group(5),
188 'status_score': match.group(6),
189 'src': match.group(7),
190 'dst': match.group(10)
191 }
192 return None
193
194def isModeExec(mode):
195 # Returns True if the given git mode represents an executable file,
196 # otherwise False.
197 return mode[-3:] == "755"
198
199def isModeExecChanged(src_mode, dst_mode):
200 return isModeExec(src_mode) != isModeExec(dst_mode)
201
202def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
203 cmd = p4_build_cmd("-G %s" % (cmd))
204 if verbose:
205 sys.stderr.write("Opening pipe: %s\n" % cmd)
206
207 # Use a temporary file to avoid deadlocks without
208 # subprocess.communicate(), which would put another copy
209 # of stdout into memory.
210 stdin_file = None
211 if stdin is not None:
212 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
213 stdin_file.write(stdin)
214 stdin_file.flush()
215 stdin_file.seek(0)
216
217 p4 = subprocess.Popen(cmd, shell=True,
218 stdin=stdin_file,
219 stdout=subprocess.PIPE)
220
221 result = []
222 try:
223 while True:
224 entry = marshal.load(p4.stdout)
225 if cb is not None:
226 cb(entry)
227 else:
228 result.append(entry)
229 except EOFError:
230 pass
231 exitCode = p4.wait()
232 if exitCode != 0:
233 entry = {}
234 entry["p4ExitCode"] = exitCode
235 result.append(entry)
236
237 return result
238
239def p4Cmd(cmd):
240 list = p4CmdList(cmd)
241 result = {}
242 for entry in list:
243 result.update(entry)
244 return result;
245
246def p4Where(depotPath):
247 if not depotPath.endswith("/"):
248 depotPath += "/"
249 depotPath = depotPath + "..."
250 outputList = p4CmdList("where %s" % depotPath)
251 output = None
252 for entry in outputList:
253 if "depotFile" in entry:
254 if entry["depotFile"] == depotPath:
255 output = entry
256 break
257 elif "data" in entry:
258 data = entry.get("data")
259 space = data.find(" ")
260 if data[:space] == depotPath:
261 output = entry
262 break
263 if output == None:
264 return ""
265 if output["code"] == "error":
266 return ""
267 clientPath = ""
268 if "path" in output:
269 clientPath = output.get("path")
270 elif "data" in output:
271 data = output.get("data")
272 lastSpace = data.rfind(" ")
273 clientPath = data[lastSpace + 1:]
274
275 if clientPath.endswith("..."):
276 clientPath = clientPath[:-3]
277 return clientPath
278
279def currentGitBranch():
280 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
281
282def isValidGitDir(path):
283 if (os.path.exists(path + "/HEAD")
284 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
285 return True;
286 return False
287
288def parseRevision(ref):
289 return read_pipe("git rev-parse %s" % ref).strip()
290
291def extractLogMessageFromGitCommit(commit):
292 logMessage = ""
293
294 ## fixme: title is first line of commit, not 1st paragraph.
295 foundTitle = False
296 for log in read_pipe_lines("git cat-file commit %s" % commit):
297 if not foundTitle:
298 if len(log) == 1:
299 foundTitle = True
300 continue
301
302 logMessage += log
303 return logMessage
304
305def extractSettingsGitLog(log):
306 values = {}
307 for line in log.split("\n"):
308 line = line.strip()
309 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
310 if not m:
311 continue
312
313 assignments = m.group(1).split (':')
314 for a in assignments:
315 vals = a.split ('=')
316 key = vals[0].strip()
317 val = ('='.join (vals[1:])).strip()
318 if val.endswith ('\"') and val.startswith('"'):
319 val = val[1:-1]
320
321 values[key] = val
322
323 paths = values.get("depot-paths")
324 if not paths:
325 paths = values.get("depot-path")
326 if paths:
327 values['depot-paths'] = paths.split(',')
328 return values
329
330def gitBranchExists(branch):
331 proc = subprocess.Popen(["git", "rev-parse", branch],
332 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
333 return proc.wait() == 0;
334
335_gitConfig = {}
336def gitConfig(key, args = None): # set args to "--bool", for instance
337 if not _gitConfig.has_key(key):
338 argsFilter = ""
339 if args != None:
340 argsFilter = "%s " % args
341 cmd = "git config %s%s" % (argsFilter, key)
342 _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
343 return _gitConfig[key]
344
345def p4BranchesInGit(branchesAreInRemotes = True):
346 branches = {}
347
348 cmdline = "git rev-parse --symbolic "
349 if branchesAreInRemotes:
350 cmdline += " --remotes"
351 else:
352 cmdline += " --branches"
353
354 for line in read_pipe_lines(cmdline):
355 line = line.strip()
356
357 ## only import to p4/
358 if not line.startswith('p4/') or line == "p4/HEAD":
359 continue
360 branch = line
361
362 # strip off p4
363 branch = re.sub ("^p4/", "", line)
364
365 branches[branch] = parseRevision(line)
366 return branches
367
368def findUpstreamBranchPoint(head = "HEAD"):
369 branches = p4BranchesInGit()
370 # map from depot-path to branch name
371 branchByDepotPath = {}
372 for branch in branches.keys():
373 tip = branches[branch]
374 log = extractLogMessageFromGitCommit(tip)
375 settings = extractSettingsGitLog(log)
376 if settings.has_key("depot-paths"):
377 paths = ",".join(settings["depot-paths"])
378 branchByDepotPath[paths] = "remotes/p4/" + branch
379
380 settings = None
381 parent = 0
382 while parent < 65535:
383 commit = head + "~%s" % parent
384 log = extractLogMessageFromGitCommit(commit)
385 settings = extractSettingsGitLog(log)
386 if settings.has_key("depot-paths"):
387 paths = ",".join(settings["depot-paths"])
388 if branchByDepotPath.has_key(paths):
389 return [branchByDepotPath[paths], settings]
390
391 parent = parent + 1
392
393 return ["", settings]
394
395def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
396 if not silent:
397 print ("Creating/updating branch(es) in %s based on origin branch(es)"
398 % localRefPrefix)
399
400 originPrefix = "origin/p4/"
401
402 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
403 line = line.strip()
404 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
405 continue
406
407 headName = line[len(originPrefix):]
408 remoteHead = localRefPrefix + headName
409 originHead = line
410
411 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
412 if (not original.has_key('depot-paths')
413 or not original.has_key('change')):
414 continue
415
416 update = False
417 if not gitBranchExists(remoteHead):
418 if verbose:
419 print "creating %s" % remoteHead
420 update = True
421 else:
422 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
423 if settings.has_key('change') > 0:
424 if settings['depot-paths'] == original['depot-paths']:
425 originP4Change = int(original['change'])
426 p4Change = int(settings['change'])
427 if originP4Change > p4Change:
428 print ("%s (%s) is newer than %s (%s). "
429 "Updating p4 branch from origin."
430 % (originHead, originP4Change,
431 remoteHead, p4Change))
432 update = True
433 else:
434 print ("Ignoring: %s was imported from %s while "
435 "%s was imported from %s"
436 % (originHead, ','.join(original['depot-paths']),
437 remoteHead, ','.join(settings['depot-paths'])))
438
439 if update:
440 system("git update-ref %s %s" % (remoteHead, originHead))
441
442def originP4BranchesExist():
443 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
444
445def p4ChangesForPaths(depotPaths, changeRange):
446 assert depotPaths
447 output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
448 for p in depotPaths]))
449
450 changes = {}
451 for line in output:
452 changeNum = int(line.split(" ")[1])
453 changes[changeNum] = True
454
455 changelist = changes.keys()
456 changelist.sort()
457 return changelist
458
459class Command:
460 def __init__(self):
461 self.usage = "usage: %prog [options]"
462 self.needsGit = True
463
464class P4Debug(Command):
465 def __init__(self):
466 Command.__init__(self)
467 self.options = [
468 optparse.make_option("--verbose", dest="verbose", action="store_true",
469 default=False),
470 ]
471 self.description = "A tool to debug the output of p4 -G."
472 self.needsGit = False
473 self.verbose = False
474
475 def run(self, args):
476 j = 0
477 for output in p4CmdList(" ".join(args)):
478 print 'Element: %d' % j
479 j += 1
480 print output
481 return True
482
483class P4RollBack(Command):
484 def __init__(self):
485 Command.__init__(self)
486 self.options = [
487 optparse.make_option("--verbose", dest="verbose", action="store_true"),
488 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
489 ]
490 self.description = "A tool to debug the multi-branch import. Don't use :)"
491 self.verbose = False
492 self.rollbackLocalBranches = False
493
494 def run(self, args):
495 if len(args) != 1:
496 return False
497 maxChange = int(args[0])
498
499 if "p4ExitCode" in p4Cmd("changes -m 1"):
500 die("Problems executing p4");
501
502 if self.rollbackLocalBranches:
503 refPrefix = "refs/heads/"
504 lines = read_pipe_lines("git rev-parse --symbolic --branches")
505 else:
506 refPrefix = "refs/remotes/"
507 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
508
509 for line in lines:
510 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
511 line = line.strip()
512 ref = refPrefix + line
513 log = extractLogMessageFromGitCommit(ref)
514 settings = extractSettingsGitLog(log)
515
516 depotPaths = settings['depot-paths']
517 change = settings['change']
518
519 changed = False
520
521 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
522 for p in depotPaths]))) == 0:
523 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
524 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
525 continue
526
527 while change and int(change) > maxChange:
528 changed = True
529 if self.verbose:
530 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
531 system("git update-ref %s \"%s^\"" % (ref, ref))
532 log = extractLogMessageFromGitCommit(ref)
533 settings = extractSettingsGitLog(log)
534
535
536 depotPaths = settings['depot-paths']
537 change = settings['change']
538
539 if changed:
540 print "%s rewound to %s" % (ref, change)
541
542 return True
543
544class P4Submit(Command):
545 def __init__(self):
546 Command.__init__(self)
547 self.options = [
548 optparse.make_option("--verbose", dest="verbose", action="store_true"),
549 optparse.make_option("--origin", dest="origin"),
550 optparse.make_option("-M", dest="detectRenames", action="store_true"),
551 ]
552 self.description = "Submit changes from git to the perforce depot."
553 self.usage += " [name of git branch to submit into perforce depot]"
554 self.interactive = True
555 self.origin = ""
556 self.detectRenames = False
557 self.verbose = False
558 self.isWindows = (platform.system() == "Windows")
559
560 def check(self):
561 if len(p4CmdList("opened ...")) > 0:
562 die("You have files opened with perforce! Close them before starting the sync.")
563
564 # replaces everything between 'Description:' and the next P4 submit template field with the
565 # commit message
566 def prepareLogMessage(self, template, message):
567 result = ""
568
569 inDescriptionSection = False
570
571 for line in template.split("\n"):
572 if line.startswith("#"):
573 result += line + "\n"
574 continue
575
576 if inDescriptionSection:
577 if line.startswith("Files:") or line.startswith("Jobs:"):
578 inDescriptionSection = False
579 else:
580 continue
581 else:
582 if line.startswith("Description:"):
583 inDescriptionSection = True
584 line += "\n"
585 for messageLine in message.split("\n"):
586 line += "\t" + messageLine + "\n"
587
588 result += line + "\n"
589
590 return result
591
592 def prepareSubmitTemplate(self):
593 # remove lines in the Files section that show changes to files outside the depot path we're committing into
594 template = ""
595 inFilesSection = False
596 for line in p4_read_pipe_lines("change -o"):
597 if line.endswith("\r\n"):
598 line = line[:-2] + "\n"
599 if inFilesSection:
600 if line.startswith("\t"):
601 # path starts and ends with a tab
602 path = line[1:]
603 lastTab = path.rfind("\t")
604 if lastTab != -1:
605 path = path[:lastTab]
606 if not path.startswith(self.depotPath):
607 continue
608 else:
609 inFilesSection = False
610 else:
611 if line.startswith("Files:"):
612 inFilesSection = True
613
614 template += line
615
616 return template
617
618 def applyCommit(self, id):
619 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
620
621 if not self.detectRenames:
622 # If not explicitly set check the config variable
623 self.detectRenames = gitConfig("git-p4.detectRenames").lower() == "true"
624
625 if self.detectRenames:
626 diffOpts = "-M"
627 else:
628 diffOpts = ""
629
630 if gitConfig("git-p4.detectCopies").lower() == "true":
631 diffOpts += " -C"
632
633 if gitConfig("git-p4.detectCopiesHarder").lower() == "true":
634 diffOpts += " --find-copies-harder"
635
636 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
637 filesToAdd = set()
638 filesToDelete = set()
639 editedFiles = set()
640 filesToChangeExecBit = {}
641 for line in diff:
642 diff = parseDiffTreeEntry(line)
643 modifier = diff['status']
644 path = diff['src']
645 if modifier == "M":
646 p4_system("edit \"%s\"" % path)
647 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
648 filesToChangeExecBit[path] = diff['dst_mode']
649 editedFiles.add(path)
650 elif modifier == "A":
651 filesToAdd.add(path)
652 filesToChangeExecBit[path] = diff['dst_mode']
653 if path in filesToDelete:
654 filesToDelete.remove(path)
655 elif modifier == "D":
656 filesToDelete.add(path)
657 if path in filesToAdd:
658 filesToAdd.remove(path)
659 elif modifier == "C":
660 src, dest = diff['src'], diff['dst']
661 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
662 if diff['src_sha1'] != diff['dst_sha1']:
663 p4_system("edit \"%s\"" % (dest))
664 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
665 p4_system("edit \"%s\"" % (dest))
666 filesToChangeExecBit[dest] = diff['dst_mode']
667 os.unlink(dest)
668 editedFiles.add(dest)
669 elif modifier == "R":
670 src, dest = diff['src'], diff['dst']
671 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
672 if diff['src_sha1'] != diff['dst_sha1']:
673 p4_system("edit \"%s\"" % (dest))
674 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
675 p4_system("edit \"%s\"" % (dest))
676 filesToChangeExecBit[dest] = diff['dst_mode']
677 os.unlink(dest)
678 editedFiles.add(dest)
679 filesToDelete.add(src)
680 else:
681 die("unknown modifier %s for %s" % (modifier, path))
682
683 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
684 patchcmd = diffcmd + " | git apply "
685 tryPatchCmd = patchcmd + "--check -"
686 applyPatchCmd = patchcmd + "--check --apply -"
687
688 if os.system(tryPatchCmd) != 0:
689 print "Unfortunately applying the change failed!"
690 print "What do you want to do?"
691 response = "x"
692 while response != "s" and response != "a" and response != "w":
693 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
694 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
695 if response == "s":
696 print "Skipping! Good luck with the next patches..."
697 for f in editedFiles:
698 p4_system("revert \"%s\"" % f);
699 for f in filesToAdd:
700 system("rm %s" %f)
701 return
702 elif response == "a":
703 os.system(applyPatchCmd)
704 if len(filesToAdd) > 0:
705 print "You may also want to call p4 add on the following files:"
706 print " ".join(filesToAdd)
707 if len(filesToDelete):
708 print "The following files should be scheduled for deletion with p4 delete:"
709 print " ".join(filesToDelete)
710 die("Please resolve and submit the conflict manually and "
711 + "continue afterwards with git-p4 submit --continue")
712 elif response == "w":
713 system(diffcmd + " > patch.txt")
714 print "Patch saved to patch.txt in %s !" % self.clientPath
715 die("Please resolve and submit the conflict manually and "
716 "continue afterwards with git-p4 submit --continue")
717
718 system(applyPatchCmd)
719
720 for f in filesToAdd:
721 p4_system("add \"%s\"" % f)
722 for f in filesToDelete:
723 p4_system("revert \"%s\"" % f)
724 p4_system("delete \"%s\"" % f)
725
726 # Set/clear executable bits
727 for f in filesToChangeExecBit.keys():
728 mode = filesToChangeExecBit[f]
729 setP4ExecBit(f, mode)
730
731 logMessage = extractLogMessageFromGitCommit(id)
732 logMessage = logMessage.strip()
733
734 template = self.prepareSubmitTemplate()
735
736 if self.interactive:
737 submitTemplate = self.prepareLogMessage(template, logMessage)
738 if os.environ.has_key("P4DIFF"):
739 del(os.environ["P4DIFF"])
740 diff = ""
741 for editedFile in editedFiles:
742 diff += p4_read_pipe("diff -du %r" % editedFile)
743
744 newdiff = ""
745 for newFile in filesToAdd:
746 newdiff += "==== new file ====\n"
747 newdiff += "--- /dev/null\n"
748 newdiff += "+++ %s\n" % newFile
749 f = open(newFile, "r")
750 for line in f.readlines():
751 newdiff += "+" + line
752 f.close()
753
754 separatorLine = "######## everything below this line is just the diff #######\n"
755
756 [handle, fileName] = tempfile.mkstemp()
757 tmpFile = os.fdopen(handle, "w+")
758 if self.isWindows:
759 submitTemplate = submitTemplate.replace("\n", "\r\n")
760 separatorLine = separatorLine.replace("\n", "\r\n")
761 newdiff = newdiff.replace("\n", "\r\n")
762 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
763 tmpFile.close()
764 mtime = os.stat(fileName).st_mtime
765 if os.environ.has_key("P4EDITOR"):
766 editor = os.environ.get("P4EDITOR")
767 else:
768 editor = read_pipe("git var GIT_EDITOR").strip()
769 system(editor + " " + fileName)
770
771 response = "y"
772 if os.stat(fileName).st_mtime <= mtime:
773 response = "x"
774 while response != "y" and response != "n":
775 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
776
777 if response == "y":
778 tmpFile = open(fileName, "rb")
779 message = tmpFile.read()
780 tmpFile.close()
781 submitTemplate = message[:message.index(separatorLine)]
782 if self.isWindows:
783 submitTemplate = submitTemplate.replace("\r\n", "\n")
784 p4_write_pipe("submit -i", submitTemplate)
785 else:
786 for f in editedFiles:
787 p4_system("revert \"%s\"" % f);
788 for f in filesToAdd:
789 p4_system("revert \"%s\"" % f);
790 system("rm %s" %f)
791
792 os.remove(fileName)
793 else:
794 fileName = "submit.txt"
795 file = open(fileName, "w+")
796 file.write(self.prepareLogMessage(template, logMessage))
797 file.close()
798 print ("Perforce submit template written as %s. "
799 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
800 % (fileName, fileName))
801
802 def run(self, args):
803 if len(args) == 0:
804 self.master = currentGitBranch()
805 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
806 die("Detecting current git branch failed!")
807 elif len(args) == 1:
808 self.master = args[0]
809 else:
810 return False
811
812 allowSubmit = gitConfig("git-p4.allowSubmit")
813 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
814 die("%s is not in git-p4.allowSubmit" % self.master)
815
816 [upstream, settings] = findUpstreamBranchPoint()
817 self.depotPath = settings['depot-paths'][0]
818 if len(self.origin) == 0:
819 self.origin = upstream
820
821 if self.verbose:
822 print "Origin branch is " + self.origin
823
824 if len(self.depotPath) == 0:
825 print "Internal error: cannot locate perforce depot path from existing branches"
826 sys.exit(128)
827
828 self.clientPath = p4Where(self.depotPath)
829
830 if len(self.clientPath) == 0:
831 print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
832 sys.exit(128)
833
834 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
835 self.oldWorkingDirectory = os.getcwd()
836
837 chdir(self.clientPath)
838 print "Synchronizing p4 checkout..."
839 p4_system("sync ...")
840
841 self.check()
842
843 commits = []
844 for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
845 commits.append(line.strip())
846 commits.reverse()
847
848 while len(commits) > 0:
849 commit = commits[0]
850 commits = commits[1:]
851 self.applyCommit(commit)
852 if not self.interactive:
853 break
854
855 if len(commits) == 0:
856 print "All changes applied!"
857 chdir(self.oldWorkingDirectory)
858
859 sync = P4Sync()
860 sync.run([])
861
862 rebase = P4Rebase()
863 rebase.rebase()
864
865 return True
866
867class P4Sync(Command):
868 delete_actions = ( "delete", "move/delete", "purge" )
869
870 def __init__(self):
871 Command.__init__(self)
872 self.options = [
873 optparse.make_option("--branch", dest="branch"),
874 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
875 optparse.make_option("--changesfile", dest="changesFile"),
876 optparse.make_option("--silent", dest="silent", action="store_true"),
877 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
878 optparse.make_option("--verbose", dest="verbose", action="store_true"),
879 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
880 help="Import into refs/heads/ , not refs/remotes"),
881 optparse.make_option("--max-changes", dest="maxChanges"),
882 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
883 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
884 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
885 help="Only sync files that are included in the Perforce Client Spec")
886 ]
887 self.description = """Imports from Perforce into a git repository.\n
888 example:
889 //depot/my/project/ -- to import the current head
890 //depot/my/project/@all -- to import everything
891 //depot/my/project/@1,6 -- to import only from revision 1 to 6
892
893 (a ... is not needed in the path p4 specification, it's added implicitly)"""
894
895 self.usage += " //depot/path[@revRange]"
896 self.silent = False
897 self.createdBranches = set()
898 self.committedChanges = set()
899 self.branch = ""
900 self.detectBranches = False
901 self.detectLabels = False
902 self.changesFile = ""
903 self.syncWithOrigin = True
904 self.verbose = False
905 self.importIntoRemotes = True
906 self.maxChanges = ""
907 self.isWindows = (platform.system() == "Windows")
908 self.keepRepoPath = False
909 self.depotPaths = None
910 self.p4BranchesInGit = []
911 self.cloneExclude = []
912 self.useClientSpec = False
913 self.clientSpecDirs = []
914
915 if gitConfig("git-p4.syncFromOrigin") == "false":
916 self.syncWithOrigin = False
917
918 #
919 # P4 wildcards are not allowed in filenames. P4 complains
920 # if you simply add them, but you can force it with "-f", in
921 # which case it translates them into %xx encoding internally.
922 # Search for and fix just these four characters. Do % last so
923 # that fixing it does not inadvertently create new %-escapes.
924 #
925 def wildcard_decode(self, path):
926 # Cannot have * in a filename in windows; untested as to
927 # what p4 would do in such a case.
928 if not self.isWindows:
929 path = path.replace("%2A", "*")
930 path = path.replace("%23", "#") \
931 .replace("%40", "@") \
932 .replace("%25", "%")
933 return path
934
935 def extractFilesFromCommit(self, commit):
936 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
937 for path in self.cloneExclude]
938 files = []
939 fnum = 0
940 while commit.has_key("depotFile%s" % fnum):
941 path = commit["depotFile%s" % fnum]
942
943 if [p for p in self.cloneExclude
944 if path.startswith (p)]:
945 found = False
946 else:
947 found = [p for p in self.depotPaths
948 if path.startswith (p)]
949 if not found:
950 fnum = fnum + 1
951 continue
952
953 file = {}
954 file["path"] = path
955 file["rev"] = commit["rev%s" % fnum]
956 file["action"] = commit["action%s" % fnum]
957 file["type"] = commit["type%s" % fnum]
958 files.append(file)
959 fnum = fnum + 1
960 return files
961
962 def stripRepoPath(self, path, prefixes):
963 if self.useClientSpec:
964
965 # if using the client spec, we use the output directory
966 # specified in the client. For example, a view
967 # //depot/foo/branch/... //client/branch/foo/...
968 # will end up putting all foo/branch files into
969 # branch/foo/
970 for val in self.clientSpecDirs:
971 if path.startswith(val[0]):
972 # replace the depot path with the client path
973 path = path.replace(val[0], val[1][1])
974 # now strip out the client (//client/...)
975 path = re.sub("^(//[^/]+/)", '', path)
976 # the rest is all path
977 return path
978
979 if self.keepRepoPath:
980 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
981
982 for p in prefixes:
983 if path.startswith(p):
984 path = path[len(p):]
985
986 return path
987
988 def splitFilesIntoBranches(self, commit):
989 branches = {}
990 fnum = 0
991 while commit.has_key("depotFile%s" % fnum):
992 path = commit["depotFile%s" % fnum]
993 found = [p for p in self.depotPaths
994 if path.startswith (p)]
995 if not found:
996 fnum = fnum + 1
997 continue
998
999 file = {}
1000 file["path"] = path
1001 file["rev"] = commit["rev%s" % fnum]
1002 file["action"] = commit["action%s" % fnum]
1003 file["type"] = commit["type%s" % fnum]
1004 fnum = fnum + 1
1005
1006 relPath = self.stripRepoPath(path, self.depotPaths)
1007
1008 for branch in self.knownBranches.keys():
1009
1010 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1011 if relPath.startswith(branch + "/"):
1012 if branch not in branches:
1013 branches[branch] = []
1014 branches[branch].append(file)
1015 break
1016
1017 return branches
1018
1019 # output one file from the P4 stream
1020 # - helper for streamP4Files
1021
1022 def streamOneP4File(self, file, contents):
1023 if file["type"] == "apple":
1024 print "\nfile %s is a strange apple file that forks. Ignoring" % \
1025 file['depotFile']
1026 return
1027
1028 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1029 relPath = self.wildcard_decode(relPath)
1030 if verbose:
1031 sys.stderr.write("%s\n" % relPath)
1032
1033 mode = "644"
1034 if isP4Exec(file["type"]):
1035 mode = "755"
1036 elif file["type"] == "symlink":
1037 mode = "120000"
1038 # p4 print on a symlink contains "target\n", so strip it off
1039 data = ''.join(contents)
1040 contents = [data[:-1]]
1041
1042 if self.isWindows and file["type"].endswith("text"):
1043 mangled = []
1044 for data in contents:
1045 data = data.replace("\r\n", "\n")
1046 mangled.append(data)
1047 contents = mangled
1048
1049 if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
1050 contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
1051 elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
1052 contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
1053
1054 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1055
1056 # total length...
1057 length = 0
1058 for d in contents:
1059 length = length + len(d)
1060
1061 self.gitStream.write("data %d\n" % length)
1062 for d in contents:
1063 self.gitStream.write(d)
1064 self.gitStream.write("\n")
1065
1066 def streamOneP4Deletion(self, file):
1067 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1068 if verbose:
1069 sys.stderr.write("delete %s\n" % relPath)
1070 self.gitStream.write("D %s\n" % relPath)
1071
1072 # handle another chunk of streaming data
1073 def streamP4FilesCb(self, marshalled):
1074
1075 if marshalled.has_key('depotFile') and self.stream_have_file_info:
1076 # start of a new file - output the old one first
1077 self.streamOneP4File(self.stream_file, self.stream_contents)
1078 self.stream_file = {}
1079 self.stream_contents = []
1080 self.stream_have_file_info = False
1081
1082 # pick up the new file information... for the
1083 # 'data' field we need to append to our array
1084 for k in marshalled.keys():
1085 if k == 'data':
1086 self.stream_contents.append(marshalled['data'])
1087 else:
1088 self.stream_file[k] = marshalled[k]
1089
1090 self.stream_have_file_info = True
1091
1092 # Stream directly from "p4 files" into "git fast-import"
1093 def streamP4Files(self, files):
1094 filesForCommit = []
1095 filesToRead = []
1096 filesToDelete = []
1097
1098 for f in files:
1099 includeFile = True
1100 for val in self.clientSpecDirs:
1101 if f['path'].startswith(val[0]):
1102 if val[1][0] <= 0:
1103 includeFile = False
1104 break
1105
1106 if includeFile:
1107 filesForCommit.append(f)
1108 if f['action'] in self.delete_actions:
1109 filesToDelete.append(f)
1110 else:
1111 filesToRead.append(f)
1112
1113 # deleted files...
1114 for f in filesToDelete:
1115 self.streamOneP4Deletion(f)
1116
1117 if len(filesToRead) > 0:
1118 self.stream_file = {}
1119 self.stream_contents = []
1120 self.stream_have_file_info = False
1121
1122 # curry self argument
1123 def streamP4FilesCbSelf(entry):
1124 self.streamP4FilesCb(entry)
1125
1126 p4CmdList("-x - print",
1127 '\n'.join(['%s#%s' % (f['path'], f['rev'])
1128 for f in filesToRead]),
1129 cb=streamP4FilesCbSelf)
1130
1131 # do the last chunk
1132 if self.stream_file.has_key('depotFile'):
1133 self.streamOneP4File(self.stream_file, self.stream_contents)
1134
1135 def commit(self, details, files, branch, branchPrefixes, parent = ""):
1136 epoch = details["time"]
1137 author = details["user"]
1138 self.branchPrefixes = branchPrefixes
1139
1140 if self.verbose:
1141 print "commit into %s" % branch
1142
1143 # start with reading files; if that fails, we should not
1144 # create a commit.
1145 new_files = []
1146 for f in files:
1147 if [p for p in branchPrefixes if f['path'].startswith(p)]:
1148 new_files.append (f)
1149 else:
1150 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1151
1152 self.gitStream.write("commit %s\n" % branch)
1153# gitStream.write("mark :%s\n" % details["change"])
1154 self.committedChanges.add(int(details["change"]))
1155 committer = ""
1156 if author not in self.users:
1157 self.getUserMapFromPerforceServer()
1158 if author in self.users:
1159 committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1160 else:
1161 committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1162
1163 self.gitStream.write("committer %s\n" % committer)
1164
1165 self.gitStream.write("data <<EOT\n")
1166 self.gitStream.write(details["desc"])
1167 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1168 % (','.join (branchPrefixes), details["change"]))
1169 if len(details['options']) > 0:
1170 self.gitStream.write(": options = %s" % details['options'])
1171 self.gitStream.write("]\nEOT\n\n")
1172
1173 if len(parent) > 0:
1174 if self.verbose:
1175 print "parent %s" % parent
1176 self.gitStream.write("from %s\n" % parent)
1177
1178 self.streamP4Files(new_files)
1179 self.gitStream.write("\n")
1180
1181 change = int(details["change"])
1182
1183 if self.labels.has_key(change):
1184 label = self.labels[change]
1185 labelDetails = label[0]
1186 labelRevisions = label[1]
1187 if self.verbose:
1188 print "Change %s is labelled %s" % (change, labelDetails)
1189
1190 files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1191 for p in branchPrefixes]))
1192
1193 if len(files) == len(labelRevisions):
1194
1195 cleanedFiles = {}
1196 for info in files:
1197 if info["action"] in self.delete_actions:
1198 continue
1199 cleanedFiles[info["depotFile"]] = info["rev"]
1200
1201 if cleanedFiles == labelRevisions:
1202 self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1203 self.gitStream.write("from %s\n" % branch)
1204
1205 owner = labelDetails["Owner"]
1206 tagger = ""
1207 if author in self.users:
1208 tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1209 else:
1210 tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1211 self.gitStream.write("tagger %s\n" % tagger)
1212 self.gitStream.write("data <<EOT\n")
1213 self.gitStream.write(labelDetails["Description"])
1214 self.gitStream.write("EOT\n\n")
1215
1216 else:
1217 if not self.silent:
1218 print ("Tag %s does not match with change %s: files do not match."
1219 % (labelDetails["label"], change))
1220
1221 else:
1222 if not self.silent:
1223 print ("Tag %s does not match with change %s: file count is different."
1224 % (labelDetails["label"], change))
1225
1226 def getUserCacheFilename(self):
1227 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1228 return home + "/.gitp4-usercache.txt"
1229
1230 def getUserMapFromPerforceServer(self):
1231 if self.userMapFromPerforceServer:
1232 return
1233 self.users = {}
1234
1235 for output in p4CmdList("users"):
1236 if not output.has_key("User"):
1237 continue
1238 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1239
1240
1241 s = ''
1242 for (key, val) in self.users.items():
1243 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1244
1245 open(self.getUserCacheFilename(), "wb").write(s)
1246 self.userMapFromPerforceServer = True
1247
1248 def loadUserMapFromCache(self):
1249 self.users = {}
1250 self.userMapFromPerforceServer = False
1251 try:
1252 cache = open(self.getUserCacheFilename(), "rb")
1253 lines = cache.readlines()
1254 cache.close()
1255 for line in lines:
1256 entry = line.strip().split("\t")
1257 self.users[entry[0]] = entry[1]
1258 except IOError:
1259 self.getUserMapFromPerforceServer()
1260
1261 def getLabels(self):
1262 self.labels = {}
1263
1264 l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1265 if len(l) > 0 and not self.silent:
1266 print "Finding files belonging to labels in %s" % `self.depotPaths`
1267
1268 for output in l:
1269 label = output["label"]
1270 revisions = {}
1271 newestChange = 0
1272 if self.verbose:
1273 print "Querying files for label %s" % label
1274 for file in p4CmdList("files "
1275 + ' '.join (["%s...@%s" % (p, label)
1276 for p in self.depotPaths])):
1277 revisions[file["depotFile"]] = file["rev"]
1278 change = int(file["change"])
1279 if change > newestChange:
1280 newestChange = change
1281
1282 self.labels[newestChange] = [output, revisions]
1283
1284 if self.verbose:
1285 print "Label changes: %s" % self.labels.keys()
1286
1287 def guessProjectName(self):
1288 for p in self.depotPaths:
1289 if p.endswith("/"):
1290 p = p[:-1]
1291 p = p[p.strip().rfind("/") + 1:]
1292 if not p.endswith("/"):
1293 p += "/"
1294 return p
1295
1296 def getBranchMapping(self):
1297 lostAndFoundBranches = set()
1298
1299 for info in p4CmdList("branches"):
1300 details = p4Cmd("branch -o %s" % info["branch"])
1301 viewIdx = 0
1302 while details.has_key("View%s" % viewIdx):
1303 paths = details["View%s" % viewIdx].split(" ")
1304 viewIdx = viewIdx + 1
1305 # require standard //depot/foo/... //depot/bar/... mapping
1306 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1307 continue
1308 source = paths[0]
1309 destination = paths[1]
1310 ## HACK
1311 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1312 source = source[len(self.depotPaths[0]):-4]
1313 destination = destination[len(self.depotPaths[0]):-4]
1314
1315 if destination in self.knownBranches:
1316 if not self.silent:
1317 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1318 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1319 continue
1320
1321 self.knownBranches[destination] = source
1322
1323 lostAndFoundBranches.discard(destination)
1324
1325 if source not in self.knownBranches:
1326 lostAndFoundBranches.add(source)
1327
1328
1329 for branch in lostAndFoundBranches:
1330 self.knownBranches[branch] = branch
1331
1332 def getBranchMappingFromGitBranches(self):
1333 branches = p4BranchesInGit(self.importIntoRemotes)
1334 for branch in branches.keys():
1335 if branch == "master":
1336 branch = "main"
1337 else:
1338 branch = branch[len(self.projectName):]
1339 self.knownBranches[branch] = branch
1340
1341 def listExistingP4GitBranches(self):
1342 # branches holds mapping from name to commit
1343 branches = p4BranchesInGit(self.importIntoRemotes)
1344 self.p4BranchesInGit = branches.keys()
1345 for branch in branches.keys():
1346 self.initialParents[self.refPrefix + branch] = branches[branch]
1347
1348 def updateOptionDict(self, d):
1349 option_keys = {}
1350 if self.keepRepoPath:
1351 option_keys['keepRepoPath'] = 1
1352
1353 d["options"] = ' '.join(sorted(option_keys.keys()))
1354
1355 def readOptions(self, d):
1356 self.keepRepoPath = (d.has_key('options')
1357 and ('keepRepoPath' in d['options']))
1358
1359 def gitRefForBranch(self, branch):
1360 if branch == "main":
1361 return self.refPrefix + "master"
1362
1363 if len(branch) <= 0:
1364 return branch
1365
1366 return self.refPrefix + self.projectName + branch
1367
1368 def gitCommitByP4Change(self, ref, change):
1369 if self.verbose:
1370 print "looking in ref " + ref + " for change %s using bisect..." % change
1371
1372 earliestCommit = ""
1373 latestCommit = parseRevision(ref)
1374
1375 while True:
1376 if self.verbose:
1377 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1378 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1379 if len(next) == 0:
1380 if self.verbose:
1381 print "argh"
1382 return ""
1383 log = extractLogMessageFromGitCommit(next)
1384 settings = extractSettingsGitLog(log)
1385 currentChange = int(settings['change'])
1386 if self.verbose:
1387 print "current change %s" % currentChange
1388
1389 if currentChange == change:
1390 if self.verbose:
1391 print "found %s" % next
1392 return next
1393
1394 if currentChange < change:
1395 earliestCommit = "^%s" % next
1396 else:
1397 latestCommit = "%s" % next
1398
1399 return ""
1400
1401 def importNewBranch(self, branch, maxChange):
1402 # make fast-import flush all changes to disk and update the refs using the checkpoint
1403 # command so that we can try to find the branch parent in the git history
1404 self.gitStream.write("checkpoint\n\n");
1405 self.gitStream.flush();
1406 branchPrefix = self.depotPaths[0] + branch + "/"
1407 range = "@1,%s" % maxChange
1408 #print "prefix" + branchPrefix
1409 changes = p4ChangesForPaths([branchPrefix], range)
1410 if len(changes) <= 0:
1411 return False
1412 firstChange = changes[0]
1413 #print "first change in branch: %s" % firstChange
1414 sourceBranch = self.knownBranches[branch]
1415 sourceDepotPath = self.depotPaths[0] + sourceBranch
1416 sourceRef = self.gitRefForBranch(sourceBranch)
1417 #print "source " + sourceBranch
1418
1419 branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1420 #print "branch parent: %s" % branchParentChange
1421 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1422 if len(gitParent) > 0:
1423 self.initialParents[self.gitRefForBranch(branch)] = gitParent
1424 #print "parent git commit: %s" % gitParent
1425
1426 self.importChanges(changes)
1427 return True
1428
1429 def importChanges(self, changes):
1430 cnt = 1
1431 for change in changes:
1432 description = p4Cmd("describe %s" % change)
1433 self.updateOptionDict(description)
1434
1435 if not self.silent:
1436 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1437 sys.stdout.flush()
1438 cnt = cnt + 1
1439
1440 try:
1441 if self.detectBranches:
1442 branches = self.splitFilesIntoBranches(description)
1443 for branch in branches.keys():
1444 ## HACK --hwn
1445 branchPrefix = self.depotPaths[0] + branch + "/"
1446
1447 parent = ""
1448
1449 filesForCommit = branches[branch]
1450
1451 if self.verbose:
1452 print "branch is %s" % branch
1453
1454 self.updatedBranches.add(branch)
1455
1456 if branch not in self.createdBranches:
1457 self.createdBranches.add(branch)
1458 parent = self.knownBranches[branch]
1459 if parent == branch:
1460 parent = ""
1461 else:
1462 fullBranch = self.projectName + branch
1463 if fullBranch not in self.p4BranchesInGit:
1464 if not self.silent:
1465 print("\n Importing new branch %s" % fullBranch);
1466 if self.importNewBranch(branch, change - 1):
1467 parent = ""
1468 self.p4BranchesInGit.append(fullBranch)
1469 if not self.silent:
1470 print("\n Resuming with change %s" % change);
1471
1472 if self.verbose:
1473 print "parent determined through known branches: %s" % parent
1474
1475 branch = self.gitRefForBranch(branch)
1476 parent = self.gitRefForBranch(parent)
1477
1478 if self.verbose:
1479 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1480
1481 if len(parent) == 0 and branch in self.initialParents:
1482 parent = self.initialParents[branch]
1483 del self.initialParents[branch]
1484
1485 self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1486 else:
1487 files = self.extractFilesFromCommit(description)
1488 self.commit(description, files, self.branch, self.depotPaths,
1489 self.initialParent)
1490 self.initialParent = ""
1491 except IOError:
1492 print self.gitError.read()
1493 sys.exit(1)
1494
1495 def importHeadRevision(self, revision):
1496 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1497
1498 details = { "user" : "git perforce import user", "time" : int(time.time()) }
1499 details["desc"] = ("Initial import of %s from the state at revision %s\n"
1500 % (' '.join(self.depotPaths), revision))
1501 details["change"] = revision
1502 newestRevision = 0
1503
1504 fileCnt = 0
1505 for info in p4CmdList("files "
1506 + ' '.join(["%s...%s"
1507 % (p, revision)
1508 for p in self.depotPaths])):
1509
1510 if 'code' in info and info['code'] == 'error':
1511 sys.stderr.write("p4 returned an error: %s\n"
1512 % info['data'])
1513 if info['data'].find("must refer to client") >= 0:
1514 sys.stderr.write("This particular p4 error is misleading.\n")
1515 sys.stderr.write("Perhaps the depot path was misspelled.\n");
1516 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
1517 sys.exit(1)
1518 if 'p4ExitCode' in info:
1519 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1520 sys.exit(1)
1521
1522
1523 change = int(info["change"])
1524 if change > newestRevision:
1525 newestRevision = change
1526
1527 if info["action"] in self.delete_actions:
1528 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1529 #fileCnt = fileCnt + 1
1530 continue
1531
1532 for prop in ["depotFile", "rev", "action", "type" ]:
1533 details["%s%s" % (prop, fileCnt)] = info[prop]
1534
1535 fileCnt = fileCnt + 1
1536
1537 details["change"] = newestRevision
1538 self.updateOptionDict(details)
1539 try:
1540 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1541 except IOError:
1542 print "IO error with git fast-import. Is your git version recent enough?"
1543 print self.gitError.read()
1544
1545
1546 def getClientSpec(self):
1547 specList = p4CmdList( "client -o" )
1548 temp = {}
1549 for entry in specList:
1550 for k,v in entry.iteritems():
1551 if k.startswith("View"):
1552
1553 # p4 has these %%1 to %%9 arguments in specs to
1554 # reorder paths; which we can't handle (yet :)
1555 if re.match('%%\d', v) != None:
1556 print "Sorry, can't handle %%n arguments in client specs"
1557 sys.exit(1)
1558
1559 if v.startswith('"'):
1560 start = 1
1561 else:
1562 start = 0
1563 index = v.find("...")
1564
1565 # save the "client view"; i.e the RHS of the view
1566 # line that tells the client where to put the
1567 # files for this view.
1568 cv = v[index+3:].strip() # +3 to remove previous '...'
1569
1570 # if the client view doesn't end with a
1571 # ... wildcard, then we're going to mess up the
1572 # output directory, so fail gracefully.
1573 if not cv.endswith('...'):
1574 print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1575 sys.exit(1)
1576 cv=cv[:-3]
1577
1578 # now save the view; +index means included, -index
1579 # means it should be filtered out.
1580 v = v[start:index]
1581 if v.startswith("-"):
1582 v = v[1:]
1583 include = -len(v)
1584 else:
1585 include = len(v)
1586
1587 temp[v] = (include, cv)
1588
1589 self.clientSpecDirs = temp.items()
1590 self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
1591
1592 def run(self, args):
1593 self.depotPaths = []
1594 self.changeRange = ""
1595 self.initialParent = ""
1596 self.previousDepotPaths = []
1597
1598 # map from branch depot path to parent branch
1599 self.knownBranches = {}
1600 self.initialParents = {}
1601 self.hasOrigin = originP4BranchesExist()
1602 if not self.syncWithOrigin:
1603 self.hasOrigin = False
1604
1605 if self.importIntoRemotes:
1606 self.refPrefix = "refs/remotes/p4/"
1607 else:
1608 self.refPrefix = "refs/heads/p4/"
1609
1610 if self.syncWithOrigin and self.hasOrigin:
1611 if not self.silent:
1612 print "Syncing with origin first by calling git fetch origin"
1613 system("git fetch origin")
1614
1615 if len(self.branch) == 0:
1616 self.branch = self.refPrefix + "master"
1617 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1618 system("git update-ref %s refs/heads/p4" % self.branch)
1619 system("git branch -D p4");
1620 # create it /after/ importing, when master exists
1621 if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1622 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1623
1624 if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1625 self.getClientSpec()
1626
1627 # TODO: should always look at previous commits,
1628 # merge with previous imports, if possible.
1629 if args == []:
1630 if self.hasOrigin:
1631 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1632 self.listExistingP4GitBranches()
1633
1634 if len(self.p4BranchesInGit) > 1:
1635 if not self.silent:
1636 print "Importing from/into multiple branches"
1637 self.detectBranches = True
1638
1639 if self.verbose:
1640 print "branches: %s" % self.p4BranchesInGit
1641
1642 p4Change = 0
1643 for branch in self.p4BranchesInGit:
1644 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
1645
1646 settings = extractSettingsGitLog(logMsg)
1647
1648 self.readOptions(settings)
1649 if (settings.has_key('depot-paths')
1650 and settings.has_key ('change')):
1651 change = int(settings['change']) + 1
1652 p4Change = max(p4Change, change)
1653
1654 depotPaths = sorted(settings['depot-paths'])
1655 if self.previousDepotPaths == []:
1656 self.previousDepotPaths = depotPaths
1657 else:
1658 paths = []
1659 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1660 for i in range(0, min(len(cur), len(prev))):
1661 if cur[i] <> prev[i]:
1662 i = i - 1
1663 break
1664
1665 paths.append (cur[:i + 1])
1666
1667 self.previousDepotPaths = paths
1668
1669 if p4Change > 0:
1670 self.depotPaths = sorted(self.previousDepotPaths)
1671 self.changeRange = "@%s,#head" % p4Change
1672 if not self.detectBranches:
1673 self.initialParent = parseRevision(self.branch)
1674 if not self.silent and not self.detectBranches:
1675 print "Performing incremental import into %s git branch" % self.branch
1676
1677 if not self.branch.startswith("refs/"):
1678 self.branch = "refs/heads/" + self.branch
1679
1680 if len(args) == 0 and self.depotPaths:
1681 if not self.silent:
1682 print "Depot paths: %s" % ' '.join(self.depotPaths)
1683 else:
1684 if self.depotPaths and self.depotPaths != args:
1685 print ("previous import used depot path %s and now %s was specified. "
1686 "This doesn't work!" % (' '.join (self.depotPaths),
1687 ' '.join (args)))
1688 sys.exit(1)
1689
1690 self.depotPaths = sorted(args)
1691
1692 revision = ""
1693 self.users = {}
1694
1695 newPaths = []
1696 for p in self.depotPaths:
1697 if p.find("@") != -1:
1698 atIdx = p.index("@")
1699 self.changeRange = p[atIdx:]
1700 if self.changeRange == "@all":
1701 self.changeRange = ""
1702 elif ',' not in self.changeRange:
1703 revision = self.changeRange
1704 self.changeRange = ""
1705 p = p[:atIdx]
1706 elif p.find("#") != -1:
1707 hashIdx = p.index("#")
1708 revision = p[hashIdx:]
1709 p = p[:hashIdx]
1710 elif self.previousDepotPaths == []:
1711 revision = "#head"
1712
1713 p = re.sub ("\.\.\.$", "", p)
1714 if not p.endswith("/"):
1715 p += "/"
1716
1717 newPaths.append(p)
1718
1719 self.depotPaths = newPaths
1720
1721
1722 self.loadUserMapFromCache()
1723 self.labels = {}
1724 if self.detectLabels:
1725 self.getLabels();
1726
1727 if self.detectBranches:
1728 ## FIXME - what's a P4 projectName ?
1729 self.projectName = self.guessProjectName()
1730
1731 if self.hasOrigin:
1732 self.getBranchMappingFromGitBranches()
1733 else:
1734 self.getBranchMapping()
1735 if self.verbose:
1736 print "p4-git branches: %s" % self.p4BranchesInGit
1737 print "initial parents: %s" % self.initialParents
1738 for b in self.p4BranchesInGit:
1739 if b != "master":
1740
1741 ## FIXME
1742 b = b[len(self.projectName):]
1743 self.createdBranches.add(b)
1744
1745 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1746
1747 importProcess = subprocess.Popen(["git", "fast-import"],
1748 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1749 stderr=subprocess.PIPE);
1750 self.gitOutput = importProcess.stdout
1751 self.gitStream = importProcess.stdin
1752 self.gitError = importProcess.stderr
1753
1754 if revision:
1755 self.importHeadRevision(revision)
1756 else:
1757 changes = []
1758
1759 if len(self.changesFile) > 0:
1760 output = open(self.changesFile).readlines()
1761 changeSet = set()
1762 for line in output:
1763 changeSet.add(int(line))
1764
1765 for change in changeSet:
1766 changes.append(change)
1767
1768 changes.sort()
1769 else:
1770 if not isinstance(self, P4Clone) and not self.p4BranchesInGit:
1771 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
1772 if self.verbose:
1773 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1774 self.changeRange)
1775 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1776
1777 if len(self.maxChanges) > 0:
1778 changes = changes[:min(int(self.maxChanges), len(changes))]
1779
1780 if len(changes) == 0:
1781 if not self.silent:
1782 print "No changes to import!"
1783 return True
1784
1785 if not self.silent and not self.detectBranches:
1786 print "Import destination: %s" % self.branch
1787
1788 self.updatedBranches = set()
1789
1790 self.importChanges(changes)
1791
1792 if not self.silent:
1793 print ""
1794 if len(self.updatedBranches) > 0:
1795 sys.stdout.write("Updated branches: ")
1796 for b in self.updatedBranches:
1797 sys.stdout.write("%s " % b)
1798 sys.stdout.write("\n")
1799
1800 self.gitStream.close()
1801 if importProcess.wait() != 0:
1802 die("fast-import failed: %s" % self.gitError.read())
1803 self.gitOutput.close()
1804 self.gitError.close()
1805
1806 return True
1807
1808class P4Rebase(Command):
1809 def __init__(self):
1810 Command.__init__(self)
1811 self.options = [ ]
1812 self.description = ("Fetches the latest revision from perforce and "
1813 + "rebases the current work (branch) against it")
1814 self.verbose = False
1815
1816 def run(self, args):
1817 sync = P4Sync()
1818 sync.run([])
1819
1820 return self.rebase()
1821
1822 def rebase(self):
1823 if os.system("git update-index --refresh") != 0:
1824 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1825 if len(read_pipe("git diff-index HEAD --")) > 0:
1826 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1827
1828 [upstream, settings] = findUpstreamBranchPoint()
1829 if len(upstream) == 0:
1830 die("Cannot find upstream branchpoint for rebase")
1831
1832 # the branchpoint may be p4/foo~3, so strip off the parent
1833 upstream = re.sub("~[0-9]+$", "", upstream)
1834
1835 print "Rebasing the current branch onto %s" % upstream
1836 oldHead = read_pipe("git rev-parse HEAD").strip()
1837 system("git rebase %s" % upstream)
1838 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1839 return True
1840
1841class P4Clone(P4Sync):
1842 def __init__(self):
1843 P4Sync.__init__(self)
1844 self.description = "Creates a new git repository and imports from Perforce into it"
1845 self.usage = "usage: %prog [options] //depot/path[@revRange]"
1846 self.options += [
1847 optparse.make_option("--destination", dest="cloneDestination",
1848 action='store', default=None,
1849 help="where to leave result of the clone"),
1850 optparse.make_option("-/", dest="cloneExclude",
1851 action="append", type="string",
1852 help="exclude depot path"),
1853 optparse.make_option("--bare", dest="cloneBare",
1854 action="store_true", default=False),
1855 ]
1856 self.cloneDestination = None
1857 self.needsGit = False
1858 self.cloneBare = False
1859
1860 # This is required for the "append" cloneExclude action
1861 def ensure_value(self, attr, value):
1862 if not hasattr(self, attr) or getattr(self, attr) is None:
1863 setattr(self, attr, value)
1864 return getattr(self, attr)
1865
1866 def defaultDestination(self, args):
1867 ## TODO: use common prefix of args?
1868 depotPath = args[0]
1869 depotDir = re.sub("(@[^@]*)$", "", depotPath)
1870 depotDir = re.sub("(#[^#]*)$", "", depotDir)
1871 depotDir = re.sub(r"\.\.\.$", "", depotDir)
1872 depotDir = re.sub(r"/$", "", depotDir)
1873 return os.path.split(depotDir)[1]
1874
1875 def run(self, args):
1876 if len(args) < 1:
1877 return False
1878
1879 if self.keepRepoPath and not self.cloneDestination:
1880 sys.stderr.write("Must specify destination for --keep-path\n")
1881 sys.exit(1)
1882
1883 depotPaths = args
1884
1885 if not self.cloneDestination and len(depotPaths) > 1:
1886 self.cloneDestination = depotPaths[-1]
1887 depotPaths = depotPaths[:-1]
1888
1889 self.cloneExclude = ["/"+p for p in self.cloneExclude]
1890 for p in depotPaths:
1891 if not p.startswith("//"):
1892 return False
1893
1894 if not self.cloneDestination:
1895 self.cloneDestination = self.defaultDestination(args)
1896
1897 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1898
1899 if not os.path.exists(self.cloneDestination):
1900 os.makedirs(self.cloneDestination)
1901 chdir(self.cloneDestination)
1902
1903 init_cmd = [ "git", "init" ]
1904 if self.cloneBare:
1905 init_cmd.append("--bare")
1906 subprocess.check_call(init_cmd)
1907
1908 if not P4Sync.run(self, depotPaths):
1909 return False
1910 if self.branch != "master":
1911 if self.importIntoRemotes:
1912 masterbranch = "refs/remotes/p4/master"
1913 else:
1914 masterbranch = "refs/heads/p4/master"
1915 if gitBranchExists(masterbranch):
1916 system("git branch master %s" % masterbranch)
1917 if not self.cloneBare:
1918 system("git checkout -f")
1919 else:
1920 print "Could not detect main branch. No checkout/master branch created."
1921
1922 return True
1923
1924class P4Branches(Command):
1925 def __init__(self):
1926 Command.__init__(self)
1927 self.options = [ ]
1928 self.description = ("Shows the git branches that hold imports and their "
1929 + "corresponding perforce depot paths")
1930 self.verbose = False
1931
1932 def run(self, args):
1933 if originP4BranchesExist():
1934 createOrUpdateBranchesFromOrigin()
1935
1936 cmdline = "git rev-parse --symbolic "
1937 cmdline += " --remotes"
1938
1939 for line in read_pipe_lines(cmdline):
1940 line = line.strip()
1941
1942 if not line.startswith('p4/') or line == "p4/HEAD":
1943 continue
1944 branch = line
1945
1946 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1947 settings = extractSettingsGitLog(log)
1948
1949 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1950 return True
1951
1952class HelpFormatter(optparse.IndentedHelpFormatter):
1953 def __init__(self):
1954 optparse.IndentedHelpFormatter.__init__(self)
1955
1956 def format_description(self, description):
1957 if description:
1958 return description + "\n"
1959 else:
1960 return ""
1961
1962def printUsage(commands):
1963 print "usage: %s <command> [options]" % sys.argv[0]
1964 print ""
1965 print "valid commands: %s" % ", ".join(commands)
1966 print ""
1967 print "Try %s <command> --help for command specific help." % sys.argv[0]
1968 print ""
1969
1970commands = {
1971 "debug" : P4Debug,
1972 "submit" : P4Submit,
1973 "commit" : P4Submit,
1974 "sync" : P4Sync,
1975 "rebase" : P4Rebase,
1976 "clone" : P4Clone,
1977 "rollback" : P4RollBack,
1978 "branches" : P4Branches
1979}
1980
1981
1982def main():
1983 if len(sys.argv[1:]) == 0:
1984 printUsage(commands.keys())
1985 sys.exit(2)
1986
1987 cmd = ""
1988 cmdName = sys.argv[1]
1989 try:
1990 klass = commands[cmdName]
1991 cmd = klass()
1992 except KeyError:
1993 print "unknown command %s" % cmdName
1994 print ""
1995 printUsage(commands.keys())
1996 sys.exit(2)
1997
1998 options = cmd.options
1999 cmd.gitdir = os.environ.get("GIT_DIR", None)
2000
2001 args = sys.argv[2:]
2002
2003 if len(options) > 0:
2004 options.append(optparse.make_option("--git-dir", dest="gitdir"))
2005
2006 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2007 options,
2008 description = cmd.description,
2009 formatter = HelpFormatter())
2010
2011 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2012 global verbose
2013 verbose = cmd.verbose
2014 if cmd.needsGit:
2015 if cmd.gitdir == None:
2016 cmd.gitdir = os.path.abspath(".git")
2017 if not isValidGitDir(cmd.gitdir):
2018 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2019 if os.path.exists(cmd.gitdir):
2020 cdup = read_pipe("git rev-parse --show-cdup").strip()
2021 if len(cdup) > 0:
2022 chdir(cdup);
2023
2024 if not isValidGitDir(cmd.gitdir):
2025 if isValidGitDir(cmd.gitdir + "/.git"):
2026 cmd.gitdir += "/.git"
2027 else:
2028 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2029
2030 os.environ["GIT_DIR"] = cmd.gitdir
2031
2032 if not cmd.run(args):
2033 parser.print_help()
2034
2035
2036if __name__ == '__main__':
2037 main()