Use bare repositories to build repo packages from the same source

4 files changed, 158 insertions, 67 deletions

diff --git a/src/common.nim b/src/common.nim
index 351dd73..b00b482 100644
--- a/src/common.nim
+++ b/src/common.nim
@@ -33,11 +33,6 @@ type
     repo: string
   ]
 
-  LookupGitResult = tuple[
-    group: LookupBaseGroup,
-    git: Option[GitRepo]
-  ]
-
 proc checkAndRefresh*(color: bool, args: seq[Argument]): tuple[code: int, args: seq[Argument]] =
   let refreshCount = args.count(%%%"refresh")
   if refreshCount > 0:
@@ -419,17 +414,70 @@ proc reloadPkgInfos*(config: Config, path: string, pkgInfos: seq[PackageInfo]):
   else:
     pkgInfos
 
+proc cloneBareRepo(config: Config, bareName: string, url: string,
+  dropPrivileges: bool): Option[string] =
+  let fullName = bareName & ".git"
+  let repoPath = repoPath(config.tmpRoot(dropPrivileges), fullName)
+  removeDirQuiet(repoPath)
+
+  if forkWait(() => (block:
+    if not dropPrivileges or dropPrivileges():
+      execResult(gitCmd, "-C", config.tmpRoot(dropPrivileges),
+        "clone", "-q", "--bare", url, fullName)
+    else:
+      quit(1))) == 0:
+    some(repoPath)
+  else:
+    removeDirQuiet(repoPath)
+    none(string)
+
+proc cloneBareRepos*(config: Config, gitRepos: seq[GitRepo],
+  progressCallback: (int, int) -> void, dropPrivileges: bool): (seq[string], seq[string]) =
+  let message = ensureTmpOrError(config, dropPrivileges)
+  if message.isSome:
+    (@[], @[message.unsafeGet])
+  else:
+    let bare = gitRepos
+      .filter(t => t.bareName.isSome)
+      .map(r => (r.bareName.unsafeGet, r.url))
+      .deduplicate
+
+    proc cloneNext(index: int, paths: List[string], messages: List[string]):
+      (List[string], List[string]) =
+      progressCallback(index, bare.len)
+
+      if index >= bare.len:
+        (paths.reversed, messages.reversed)
+      else:
+        let (bareName, url) = bare[index]
+        let repoPath = cloneBareRepo(config, bareName, url, dropPrivileges)
+        if repoPath.isSome:
+          cloneNext(index + 1, repoPath.unsafeGet ^& paths, messages)
+        else:
+          let message = tr"$#: failed to clone git repository" % [bareName]
+          cloneNext(index + 1, paths, message ^& messages)
+
+    let (paths, messages) = cloneNext(0, nil, nil)
+    (toSeq(paths.items), toSeq(messages.items))
+
 proc clonePackageRepoInternal(config: Config, base: string, version: string,
   git: GitRepo, dropPrivileges: bool): Option[string] =
-  let tmpRoot = config.tmpRoot(dropPrivileges)
-  let repoPath = repoPath(tmpRoot, base)
+  let repoPath = repoPath(config.tmpRoot(dropPrivileges), base)
   removeDirQuiet(repoPath)
 
+  let url = if git.bareName.isSome:
+      repoPath(config.tmpRoot(dropPrivileges), git.bareName.unsafeGet & ".git")
+    else:
+      git.url
+
   if forkWait(() => (block:
     if not dropPrivileges or dropPrivileges():
-      execResult(gitCmd, "-C", tmpRoot,
-        "clone", "-q", git.url, "-b", git.branch,
-        "--single-branch", base)
+      if git.branch.isSome:
+        execResult(gitCmd, "-C", config.tmpRoot(dropPrivileges),
+          "clone", "-q", url, "-b", git.branch.unsafeGet, "--single-branch", base)
+      else:
+        execResult(gitCmd, "-C", config.tmpRoot(dropPrivileges),
+          "clone", "-q", url, "--single-branch", base)
     else:
       quit(1))) == 0:
     let commit = bisectVersion(repoPath, config.debug, none(string),
@@ -466,7 +514,7 @@ proc clonePackageRepo*(config: Config, base: string, version: string,
 proc obtainBuildPkgInfosInternal(config: Config, bases: seq[LookupBaseGroup],
   pacmanTargetNames: seq[string], progressCallback: (int, int) -> void, dropPrivileges: bool):
   (seq[PackageInfo], seq[string], seq[string]) =
-  let lookupResults: seq[LookupGitResult] = bases
+  let lookupResults: seq[tuple[group: LookupBaseGroup, git: Option[GitRepo]]] = bases
     .map(b => (b, lookupGitRepo(b.repo, b.base, b.arch)))
   let notFoundRepos = lookupResults.filter(r => r.git.isNone)
 
@@ -478,45 +526,56 @@ proc obtainBuildPkgInfosInternal(config: Config, bases: seq[LookupBaseGroup],
     if message.isSome:
       (@[], @[], @[message.unsafeGet])
     else:
-      proc findCommitAndGetSrcInfo(base: string, version: string,
-        repo: string, git: GitRepo): tuple[pkgInfos: seq[PackageInfo], path: Option[string]] =
-        let repoPath = clonePackageRepoInternal(config, base, version, git, dropPrivileges)
+      let (barePaths, berrors) = cloneBareRepos(config, lookupResults.map(r => r.git.unsafeGet),
+        proc (progress: int, count: int) = progressCallback(progress, count + lookupResults.len),
+        dropPrivileges)
 
-        if repoPath.isSome:
-          let srcInfo = obtainSrcInfo(repoPath.unsafeGet & "/" & git.path)
-          let pkgInfos = parseSrcInfo(repo, srcInfo, config.arch,
-            git.url, some(git.path))
-            .filter(i => i.version == version)
-          (pkgInfos, repoPath)
-        else:
-          (newSeq[PackageInfo](), none(string))
-
-      progressCallback(0, lookupResults.len)
-      let (pkgInfosWithPathsReversed, _) = lookupResults.foldl(block:
-        let (list, index) = a
-        let res = findCommitAndGetSrcInfo(b.group.base, b.group.version,
-          b.group.repo, b.git.get) ^& list
-        progressCallback(index + 1, lookupResults.len)
-        (res, index + 1),
-        (list[tuple[pkgInfos: seq[PackageInfo], path: Option[string]]](), 0))
-
-      let pkgInfosWithPaths = pkgInfosWithPathsReversed.reversed
-      let pkgInfos = lc[x | (y <- pkgInfosWithPaths, x <- y.pkgInfos), PackageInfo]
-      let paths = lc[x | (y <- pkgInfosWithPaths, x <- y.path), string]
-
-      let pkgInfosTable = pkgInfos.map(i => (i.name, i)).toTable
-
-      let foundPkgInfos = lc[x | (y <- pacmanTargetNames,
-        x <- pkgInfosTable.opt(y)), PackageInfo]
-      let errorMessages = pacmanTargetNames
-        .filter(n => not pkgInfosTable.hasKey(n))
-        .map(n => tr"$#: failed to get package info" % [n])
-
-      if errorMessages.len > 0:
-        for path in paths:
+      if berrors.len > 0:
+        for path in barePaths:
           removeDirQuiet(path)
-      discard rmdir(config.tmpRoot(dropPrivileges))
-      (foundPkgInfos, paths, errorMessages)
+        discard rmdir(config.tmpRoot(dropPrivileges))
+        (newSeq[PackageInfo](), barePaths, berrors)
+      else:
+        proc findCommitAndGetSrcInfo(base: string, version: string,
+          repo: string, git: GitRepo): tuple[pkgInfos: seq[PackageInfo], path: Option[string]] =
+          let repoPath = clonePackageRepoInternal(config, base, version, git, dropPrivileges)
+
+          if repoPath.isSome:
+            let srcInfo = obtainSrcInfo(repoPath.unsafeGet & "/" & git.path)
+            let pkgInfos = parseSrcInfo(repo, srcInfo, config.arch,
+              git.url, some(git.path))
+              .filter(i => i.version == version)
+            (pkgInfos, repoPath)
+          else:
+            (newSeq[PackageInfo](), none(string))
+
+        let (pkgInfosWithPathsReversed, _) = lookupResults.foldl(block:
+          let (list, index) = a
+          let res = findCommitAndGetSrcInfo(b.group.base, b.group.version,
+            b.group.repo, b.git.unsafeGet) ^& list
+          progressCallback(barePaths.len + index + 1, barePaths.len + lookupResults.len)
+          (res, index + 1),
+          (list[tuple[pkgInfos: seq[PackageInfo], path: Option[string]]](), 0))
+
+        let pkgInfosWithPaths = pkgInfosWithPathsReversed.reversed
+        let pkgInfos = lc[x | (y <- pkgInfosWithPaths, x <- y.pkgInfos), PackageInfo]
+        let paths = lc[x | (y <- pkgInfosWithPaths, x <- y.path), string]
+
+        let pkgInfosTable = pkgInfos.map(i => (i.name, i)).toTable
+
+        let foundPkgInfos = lc[x | (y <- pacmanTargetNames,
+          x <- pkgInfosTable.opt(y)), PackageInfo]
+        let errorMessages = pacmanTargetNames
+          .filter(n => not pkgInfosTable.hasKey(n))
+          .map(n => tr"$#: failed to get package info" % [n])
+
+        if errorMessages.len > 0:
+          for path in barePaths:
+            removeDirQuiet(path)
+          for path in paths:
+            removeDirQuiet(path)
+        discard rmdir(config.tmpRoot(dropPrivileges))
+        (foundPkgInfos, barePaths & paths, errorMessages)
 
 proc obtainBuildPkgInfos*[T: RpcPackageInfo](config: Config,
   pacmanTargets: seq[FullPackageTarget[T]], progressCallback: (int, int) -> void,
diff --git a/src/feature/syncinstall.nim b/src/feature/syncinstall.nim
index ab01df1..61bd857 100644
--- a/src/feature/syncinstall.nim
+++ b/src/feature/syncinstall.nim
@@ -37,17 +37,20 @@ type
 proc groupsSeq(pkg: ptr AlpmPackage): seq[string] =
   toSeq(pkg.groups.items).map(s => $s)
 
-proc createCloneProgress(config: Config, count: int, printMode: bool):
+proc createCloneProgress(config: Config, count: int, flexible: bool, printMode: bool):
   (proc (update: int, terminate: int) {.closure.}, proc {.closure.}) =
   if count >= 1 and not printMode:
     let (update, terminate) = printProgressShare(config.progressBar, tr"cloning repositories")
     update(0, count)
 
-    proc cloneUpdate(progress: int, newCount: int) {.closure.} =
-      # newCount can be < count if some packages were not found
-      update(max(count - newCount + progress, 0), count)
+    if flexible:
+      proc cloneUpdate(progress: int, newCount: int) {.closure.} =
+        # newCount can be < count if some packages were not found
+        update(max(count - newCount + progress, 0), count)
 
-    (cloneUpdate, terminate)
+      (cloneUpdate, terminate)
+    else:
+      (update, terminate)
   else:
     (proc (a: int, b: int) {.closure.} = discard, proc {.closure.} = discard)
 
@@ -185,7 +188,7 @@ proc findDependencies(config: Config, handle: ptr AlpmHandle, dbs: seq[ptr AlpmD
 
   let (aurSuccess, aurFail, newPaths, newAdditionalPkgInfos) =
     if not noaur and aurCheck.len > 0: (block:
-      let (update, terminate) = createCloneProgress(config, aurCheck.len, printMode)
+      let (update, terminate) = createCloneProgress(config, aurCheck.len, true, printMode)
       try:
         withAur():
           let (pkgInfos, additionalPkgInfos, paths) = if printMode: (block:
@@ -1028,7 +1031,7 @@ proc obtainAurPackageInfos(config: Config, rpcInfos: seq[RpcPackageInfo],
   let upgradeRpcInfos = upgradeStructs.filter(p => p.needed).map(p => p.rpcInfo)
   let fullRpcInfos = targetRpcInfos & upgradeRpcInfos
 
-  let (update, terminate) = createCloneProgress(config, fullRpcInfos.len, printMode)
+  let (update, terminate) = createCloneProgress(config, fullRpcInfos.len, true, printMode)
 
   let (pkgInfos, additionalPkgInfos, paths, errors) = if printMode: (block:
       let (pkgInfos, additionalPkgInfos, aerrors) =
@@ -1077,7 +1080,7 @@ proc obtainPacmanBuildTargets(config: Config, pacmanTargets: seq[FullPackageTarg
 
   let (buildPkgInfos, buildPaths, obtainErrorMessages) = if checkPacmanBuildPkgInfos: (block:
       echo(tr"checking official repositories...")
-      let (update, terminate) = createCloneProgress(config, pacmanTargets.len, printMode)
+      let (update, terminate) = createCloneProgress(config, pacmanTargets.len, false, printMode)
       let res = obtainBuildPkgInfos[PackageInfo](config, pacmanTargets, update, true)
       terminate()
       res)
diff --git a/src/feature/syncsource.nim b/src/feature/syncsource.nim
index fba7062..f7e2524 100644
--- a/src/feature/syncsource.nim
+++ b/src/feature/syncsource.nim
@@ -33,10 +33,14 @@ proc getFilesOrClear(base: string, repoPath: string, gitSubdir: Option[string]):
     (newSeq[string](), some(tr"$#: failed to clone git repository" % [base]))
 
 proc cloneRepositories(config: Config, targets: seq[BaseTarget],
-  update: (int, int) -> void): (List[CloneResult], List[string]) =
+  update: (int, int) -> void): (List[CloneResult], seq[string], seq[string]) =
+  let (barePaths, berrors) = cloneBareRepos(config,
+    targets.filter(t => t.gitRepo.isSome).map(t => t.gitRepo.unsafeGet),
+    proc (progress: int, count: int) = update(progress, count + targets.len), false)
+
   proc cloneNext(index: int, results: List[CloneResult], messages: List[string]):
     (List[CloneResult], List[string]) =
-    update(index, targets.len)
+    update(barePaths.len + index, barePaths.len + targets.len)
 
     if index >= targets.len:
       (results.reversed, messages.reversed)
@@ -76,7 +80,11 @@ proc cloneRepositories(config: Config, targets: seq[BaseTarget],
         let message = tr"$#: repository not found" % [target.base]
         cloneNext(index + 1, results, message ^& messages)
 
-  cloneNext(0, nil, nil)
+  if berrors.len > 0:
+    (nil, barePaths, berrors)
+  else:
+    let (results, cerrors) = cloneNext(0, nil, nil)
+    (results, barePaths, toSeq(cerrors.items))
 
 proc copyFiles(config: Config, quiet: bool, results: seq[CloneResult]): List[string] =
   proc copyNext(index: int, messages: List[string]): List[string] =
@@ -128,7 +136,7 @@ proc cloneAndCopy(config: Config, quiet: bool,
     else:
       printProgressShare(config.progressBar, tr"cloning repositories")
 
-  let (results, rerrors) = cloneRepositories(config, baseTargets, update)
+  let (results, barePaths, rerrors) = cloneRepositories(config, baseTargets, update)
   terminate()
   for e in rerrors: printError(config.color, e)
 
@@ -137,6 +145,8 @@ proc cloneAndCopy(config: Config, quiet: bool,
 
   for result in results:
     removeDirQuiet(result.path)
+  for path in barePaths:
+    removeDirQuiet(path)
   discard rmdir(config.tmpRootCurrent)
 
   if rerrors != nil and cerrors != nil:
diff --git a/src/package.nim b/src/package.nim
index 3884a34..c9ea5b2 100644
--- a/src/package.nim
+++ b/src/package.nim
@@ -53,7 +53,8 @@ type
 
   GitRepo* = tuple[
     url: string,
-    branch: string,
+    bareName: Option[string],
+    branch: Option[string],
     path: string
   ]
 
@@ -69,12 +70,12 @@ const
   packageRepos: seq[PackageRepo] = @[
     (["arch"].toSet,
       ["core", "extra", "testing"].toSet,
-      ("https://git.archlinux.org/svntogit/packages.git",
-        "packages/${BASE}", "repos/${REPO}-${ARCH}")),
+      ("https://git.archlinux.org/svntogit/packages.git", none(string),
+        some("packages/${BASE}"), "repos/${REPO}-${ARCH}")),
     (["arch"].toSet,
       ["community", "community-testing", "multilib", "multilib-testing"].toSet,
-      ("https://git.archlinux.org/svntogit/community.git",
-        "packages/${BASE}", "repos/${REPO}-${ARCH}"))
+      ("https://git.archlinux.org/svntogit/community.git", none(string),
+        some("packages/${BASE}"), "repos/${REPO}-${ARCH}"))
   ]
 
 static:
@@ -89,6 +90,23 @@ static:
         raise newException(SystemError,
           "only single matching repo available: " & os & ":" & repo)
 
+  # test unique url <> bareName links
+  let bareNameToUrl = lc[(x, r.git.url) |
+    (r <- packageRepos, x <- r.git.bareName), (string, string)].toTable
+  let urlToBareName = lc[(r.git.url, x) |
+    (r <- packageRepos, x <- r.git.bareName), (string, string)].toTable
+
+  template testBareNamesAndUrls(m1: untyped, m2: untyped) =
+    for x1, x2 in m1:
+      try:
+        if m2[x2] != x1:
+          raise newException(SystemError, "")
+      except:
+        raise newException(SystemError, "Invalid url <> bareName links")
+
+  testBareNamesAndUrls(bareNameToUrl, urlToBareName)
+  testBareNamesAndUrls(urlToBareName, bareNameToUrl)
+
 proc readOsId: Option[string] =
   var file: File
   if file.open("/usr/lib/os-release"):
@@ -108,7 +126,7 @@ proc readOsId: Option[string] =
 let osId = readOsId()
 
 proc lookupGitRepo*(repo: string, base: string, arch: string): Option[GitRepo] =
-  template replaceAll(gitPart: string): string =
+  proc replaceAll(gitPart: string): string =
     gitPart
       .replace("${REPO}", repo)
       .replace("${BASE}", base)
@@ -116,7 +134,8 @@ proc lookupGitRepo*(repo: string, base: string, arch: string): Option[GitRepo] =
 
   packageRepos
     .filter(pr => osId.isSome and osid.unsafeGet in pr.os and repo in pr.repo)
-    .map(pr => (pr.git.url.replaceAll, pr.git.branch.replaceAll, pr.git.path.replaceAll))
+    .map(pr => (pr.git.url.replaceAll, pr.git.bareName,
+      pr.git.branch.map(replaceAll), pr.git.path.replaceAll))
     .optFirst
 
 template repoPath*(tmpRoot: string, base: string): string =