Add PKGBUILD retrieving

3 files changed, 187 insertions, 49 deletions

diff --git a/src/feature/syncinfo.nim b/src/feature/syncinfo.nim
index 42d7e91..b0765f4 100644
--- a/src/feature/syncinfo.nim
+++ b/src/feature/syncinfo.nim
@@ -101,7 +101,7 @@ proc handleTarget(config: Config, padding: int, args: seq[Argument],
 
 proc handleSyncInfo*(args: seq[Argument], config: Config): int =
   let (_, callArgs) = checkAndRefresh(config.color, args)
-  let targets = args.packageTargets
+  let targets = args.packageTargets(false)
 
   let (syncTargets, checkAurNames) = withAlpm(config.root, config.db,
     config.dbs, config.arch, handle, dbs, errors):
diff --git a/src/feature/syncinstall.nim b/src/feature/syncinstall.nim
index c214198..ab01df1 100644
--- a/src/feature/syncinstall.nim
+++ b/src/feature/syncinstall.nim
@@ -197,7 +197,7 @@ proc findDependencies(config: Config, handle: ptr AlpmHandle, dbs: seq[ptr AlpmD
               let (rpcInfos, aerrors) = getRpcPackageInfos(aurCheck.map(r => r.name))
               for e in aerrors: printError(config.color, e)
               let (pkgInfos, additionalPkgInfos, paths, cerrors) =
-                cloneAurReposWithPackageInfos(config, rpcInfos, not printMode, update)
+                cloneAurReposWithPackageInfos(config, rpcInfos, not printMode, update, true)
               for e in cerrors: printError(config.color, e)
               (pkgInfos, additionalPkgInfos, paths))
 
@@ -247,30 +247,7 @@ proc findDependencies(config: Config, handle: ptr AlpmHandle,
 template clearPaths(paths: untyped) =
   for path in paths:
     removeDirQuiet(path)
-  discard rmdir(config.tmpRoot)
-
-proc filterNotFoundSyncTargetsInternal(syncTargets: seq[SyncPackageTarget],
-  pkgInfoReferencesTable: Table[string, PackageReference],
-  upToDateNeededTable: Table[string, PackageReference]): seq[SyncPackageTarget] =
-  # collect packages which were found neither in sync DB nor in AUR
-  syncTargets.filter(t => not (upToDateNeededTable.opt(t.reference.name)
-    .map(r => t.reference.isProvidedBy(r)).get(false)) and t.foundInfos.len == 0 and
-    not (t.isAurTargetSync and pkgInfoReferencesTable.opt(t.reference.name)
-    .map(r => t.reference.isProvidedBy(r)).get(false)))
-
-proc filterNotFoundSyncTargets[T: RpcPackageInfo](syncTargets: seq[SyncPackageTarget],
-  pkgInfos: seq[T], upToDateNeededTable: Table[string, PackageReference]): seq[SyncPackageTarget] =
-  let pkgInfoReferencesTable = pkgInfos.map(i => (i.name, i.toPackageReference)).toTable
-  filterNotFoundSyncTargetsInternal(syncTargets, pkgInfoReferencesTable, upToDateNeededTable)
-
-proc printSyncNotFound(config: Config, notFoundTargets: seq[SyncPackageTarget]) =
-  let dbs = config.dbs.toSet
-
-  for target in notFoundTargets:
-    if target.repo.isNone or target.repo == some("aur") or target.repo.unsafeGet in dbs:
-      printError(config.color, trp("target not found: %s\n") % [$target.reference])
-    else:
-      printError(config.color, trp("database not found: %s\n") % [target.repo.unsafeGet])
+  discard rmdir(config.tmpRootInitial)
 
 proc printUnsatisfied(config: Config,
   satisfied: Table[PackageReference, SatisfyResult], unsatisfied: seq[PackageReference]) =
@@ -328,19 +305,7 @@ proc editLoop(config: Config, base: string, repoPath: string, gitSubdir: Option[
     else:
       res
 
-  let rawFiles = if gitSubdir.isSome:
-      forkWaitRedirect(() => (block:
-        dropPrivilegesAndChdir(none(string)):
-          execResult(gitCmd, "-C", repoPath, "ls-tree", "-r", "--name-only", "@",
-            gitSubdir.unsafeGet & "/")))
-        .output
-        .map(s => s[gitSubdir.unsafeGet.len + 1 .. ^1])
-    else:
-      forkWaitRedirect(() => (block:
-        dropPrivilegesAndChdir(none(string)):
-          execResult(gitCmd, "-C", repoPath, "ls-tree", "-r", "--name-only", "@")))
-        .output
-
+  let rawFiles = getGitFiles(repoPath, gitSubdir, true)
   let files = ("PKGBUILD" & rawFiles.filter(x => x != ".SRCINFO")).deduplicate
 
   proc editFileLoopAll(index: int): char =
@@ -355,7 +320,7 @@ proc editLoop(config: Config, base: string, repoPath: string, gitSubdir: Option[
 proc buildLoop(config: Config, pkgInfos: seq[PackageInfo], noconfirm: bool,
   noextract: bool): (Option[BuildResult], int, bool) =
   let base = pkgInfos[0].base
-  let repoPath = repoPath(config.tmpRoot, base)
+  let repoPath = repoPath(config.tmpRootInitial, base)
   let gitSubdir = pkgInfos[0].gitSubdir
   let buildPath = buildPath(repoPath, gitSubdir)
 
@@ -365,7 +330,7 @@ proc buildLoop(config: Config, pkgInfos: seq[PackageInfo], noconfirm: bool,
     else:
       confFileEnv
 
-  let workConfFile = config.tmpRoot & "/makepkg.conf"
+  let workConfFile = config.tmpRootInitial & "/makepkg.conf"
 
   let workConfFileCopySuccess = try:
     copyFile(confFile, workConfFile)
@@ -377,7 +342,7 @@ proc buildLoop(config: Config, pkgInfos: seq[PackageInfo], noconfirm: bool,
         file.writeLine("# PAKKU OVERRIDES")
         file.writeLine('#'.repeat(73))
         file.writeLine("CARCH=" & config.arch.bashEscape)
-        file.writeLine("PKGDEST=" & config.tmpRoot.bashEscape)
+        file.writeLine("PKGDEST=" & config.tmpRootInitial.bashEscape)
       finally:
         file.close()
     true
@@ -458,7 +423,7 @@ proc buildLoop(config: Config, pkgInfos: seq[PackageInfo], noconfirm: bool,
 proc buildFromSources(config: Config, commonArgs: seq[Argument],
   pkgInfos: seq[PackageInfo], noconfirm: bool): (Option[BuildResult], int) =
   let base = pkgInfos[0].base
-  let repoPath = repoPath(config.tmpRoot, base)
+  let repoPath = repoPath(config.tmpRootInitial, base)
   let gitSubdir = pkgInfos[0].gitSubdir
 
   proc loop(noextract: bool, showEditLoop: bool): (Option[BuildResult], int) =
@@ -536,7 +501,7 @@ proc installGroupFromSources(config: Config, commonArgs: seq[Argument],
 
   proc formatArchiveFile(pkgInfo: PackageInfo, ext: string): string =
     let arch = if config.arch in pkgInfo.archs: config.arch else: "any"
-    config.tmpRoot & "/" & pkgInfo.name & "-" & pkgInfo.version & "-" & arch & ext
+    config.tmpRootInitial & "/" & pkgInfo.name & "-" & pkgInfo.version & "-" & arch & ext
 
   let allFiles = lc[(r.name, formatArchiveFile(r.pkgInfo, br.ext)) |
     (br <- buildResults, r <- br.replacePkgInfos), tuple[name: Option[string], file: string]]
@@ -554,7 +519,7 @@ proc installGroupFromSources(config: Config, commonArgs: seq[Argument],
           discard
 
     if not clear:
-      printWarning(config.color, tr"packages are saved to '$#'" % [config.tmpRoot])
+      printWarning(config.color, tr"packages are saved to '$#'" % [config.tmpRootInitial])
 
   if buildCode != 0:
     handleTmpRoot(true)
@@ -654,7 +619,7 @@ proc confirmViewAndImportKeys(config: Config, basePackages: seq[seq[seq[PackageI
         if index < flatBasePackages.len:
           let pkgInfos = flatBasePackages[index]
           let base = pkgInfos[0].base
-          let repoPath = repoPath(config.tmpRoot, base)
+          let repoPath = repoPath(config.tmpRootInitial, base)
 
           let aur = pkgInfos[0].repo == "aur"
 
@@ -1072,7 +1037,7 @@ proc obtainAurPackageInfos(config: Config, rpcInfos: seq[RpcPackageInfo],
     else: (block:
       let (rpcInfos, aerrors) = getRpcPackageInfos(fullRpcInfos.map(i => i.name))
       let (pkgInfos, additionalPkgInfos, paths, cerrors) =
-        cloneAurReposWithPackageInfos(config, rpcInfos, not printMode, update)
+        cloneAurReposWithPackageInfos(config, rpcInfos, not printMode, update, true)
       (pkgInfos, additionalPkgInfos, paths, (toSeq(aerrors.items) & cerrors).deduplicate))
 
   terminate()
@@ -1113,7 +1078,7 @@ proc obtainPacmanBuildTargets(config: Config, pacmanTargets: seq[FullPackageTarg
   let (buildPkgInfos, buildPaths, obtainErrorMessages) = if checkPacmanBuildPkgInfos: (block:
       echo(tr"checking official repositories...")
       let (update, terminate) = createCloneProgress(config, pacmanTargets.len, printMode)
-      let res = obtainBuildPkgInfos[PackageInfo](config, pacmanTargets, update)
+      let res = obtainBuildPkgInfos[PackageInfo](config, pacmanTargets, update, true)
       terminate()
       res)
     else:
@@ -1259,7 +1224,7 @@ proc handleSyncInstall*(args: seq[Argument], config: Config): int =
       arg.matchOption(%%%"noconfirm")).optLast
     .map(arg => arg.key == "noconfirm").get(false)
 
-  let targets = args.packageTargets
+  let targets = args.packageTargets(false)
 
   withAur():
     let (code, installed, foreignUpgrade, targetNamesSet, pacmanTargets,
diff --git a/src/feature/syncsource.nim b/src/feature/syncsource.nim
new file mode 100644
index 0000000..fba7062
--- /dev/null
+++ b/src/feature/syncsource.nim
@@ -0,0 +1,173 @@
+import
+  future, options, os, posix, sequtils, strutils, tables,
+  "../args", "../aur", "../common", "../config", "../format", "../lists",
+    "../package", "../pacman", "../utils",
+  "../wrapper/alpm"
+
+type
+  BaseTarget = tuple[
+    base: string,
+    version: string,
+    destination: string,
+    aurGitUrl: Option[string],
+    gitRepo: Option[GitRepo]
+  ]
+
+  CloneResult = tuple[
+    base: string,
+    path: string,
+    files: seq[string],
+    destination: string
+  ]
+
+proc getFilesOrClear(base: string, repoPath: string, gitSubdir: Option[string]):
+  (seq[string], Option[string]) =
+  let rawFiles = getGitFiles(repoPath, gitSubdir, false)
+    .filter(f => f != ".gitignore" and f != ".SRCINFO" and f.find('/') < 0)
+    .map(f => gitSubdir.map(s => s & "/" & f).get(f))
+
+  if rawFiles.len > 0:
+    (rawFiles, none(string))
+  else:
+    removeDirQuiet(repoPath)
+    (newSeq[string](), some(tr"$#: failed to clone git repository" % [base]))
+
+proc cloneRepositories(config: Config, targets: seq[BaseTarget],
+  update: (int, int) -> void): (List[CloneResult], List[string]) =
+  proc cloneNext(index: int, results: List[CloneResult], messages: List[string]):
+    (List[CloneResult], List[string]) =
+    update(index, targets.len)
+
+    if index >= targets.len:
+      (results.reversed, messages.reversed)
+    else:
+      let target = targets[index]
+      let repoPath = repoPath(config.tmpRootCurrent, target.base)
+      removeDirQuiet(repoPath)
+
+      if target.aurGitUrl.isSome:
+        let (cloneCode, cerror) = cloneAurRepo(config,
+          target.base, target.aurGitUrl.unsafeGet, false)
+
+        if cloneCode != 0:
+          cloneNext(index + 1, results, toSeq(cerror.items) ^& messages)
+        else:
+          let (files, ferror) = getFilesOrClear(target.base, repoPath, none(string))
+          if ferror.isSome:
+            cloneNext(index + 1, results, ferror.unsafeGet ^& messages)
+          else:
+            cloneNext(index + 1, (target.base, repoPath, files,
+              target.destination) ^& results, messages)
+      elif target.gitRepo.isSome:
+        let gitRepo = target.gitRepo.unsafeGet
+        let cerror = clonePackageRepo(config, target.base,
+          target.version, gitRepo, false)
+
+        if cerror.isSome:
+          cloneNext(index + 1, results, cerror.unsafeGet ^& messages)
+        else:
+          let (files, ferror) = getFilesOrClear(target.base, repoPath, some(gitRepo.path))
+          if ferror.isSome:
+            cloneNext(index + 1, results, ferror.unsafeGet ^& messages)
+          else:
+            cloneNext(index + 1, (target.base, repoPath, files,
+              target.destination) ^& results, messages)
+      else:
+        let message = tr"$#: repository not found" % [target.base]
+        cloneNext(index + 1, results, message ^& messages)
+
+  cloneNext(0, nil, nil)
+
+proc copyFiles(config: Config, quiet: bool, results: seq[CloneResult]): List[string] =
+  proc copyNext(index: int, messages: List[string]): List[string] =
+    if index >= results.len:
+      messages.reversed
+    else:
+      let res = results[index]
+      discard mkdir(res.destination, 0o755)
+
+      let error = try:
+        for f in res.files:
+          let index = f.rfind('/')
+          let name = if index >= 0: f[index + 1 .. ^1] else: f
+          let dest = if res.destination == ".": name else: res.destination & "/" & name
+          moveFile(res.path & "/" & f, dest)
+          printFile(config.color, quiet, res.base, dest)
+        none(string)
+      except OSError:
+        some(tr"$#: failed to move files" % [res.base])
+
+      copyNext(index + 1, toSeq(error.items) ^& messages)
+
+  copyNext(0, nil)
+
+proc cloneAndCopy(config: Config, quiet: bool,
+  fullTargets: seq[FullPackageTarget[RpcPackageInfo]]): int =
+  let baseTargets = fullTargets.foldl(block:
+    let bases = a.map(x => x.base)
+    if b.isAurTargetFull:
+      let rpcInfo = b.pkgInfo.get
+      if rpcInfo.base in bases:
+        a
+      else:
+        a & (rpcInfo.base, rpcInfo.version, b.destination.get(rpcInfo.base),
+          some(rpcInfo.gitUrl), none(GitRepo))
+    else:
+      let foundInfo = b.foundInfos[0]
+      let pkg = foundInfo.pkg.get
+      if pkg.base in bases:
+        a
+      else:
+        let git = lookupGitRepo(foundInfo.repo, pkg.base, pkg.arch.get)
+        a & (pkg.base, pkg.version, b.destination.get(pkg.base),
+          none(string), git),
+    newSeq[BaseTarget]())
+
+  let (update, terminate) = if quiet:
+      (proc (a: int, b: int) {.closure.} = discard, proc () {.closure.} = discard)
+    else:
+      printProgressShare(config.progressBar, tr"cloning repositories")
+
+  let (results, rerrors) = cloneRepositories(config, baseTargets, update)
+  terminate()
+  for e in rerrors: printError(config.color, e)
+
+  let cerrors = copyFiles(config, quiet, toSeq(results.items))
+  for e in cerrors: printError(config.color, e)
+
+  for result in results:
+    removeDirQuiet(result.path)
+  discard rmdir(config.tmpRootCurrent)
+
+  if rerrors != nil and cerrors != nil:
+    1
+  else:
+    0
+
+proc handleSyncSource*(args: seq[Argument], config: Config): int =
+  discard checkAndRefresh(config.color, args)
+
+  let quiet = args.check(%%%"quiet")
+  let targets = args.packageTargets(true)
+
+  if targets.len == 0:
+    printError(config.color, trp("no targets specified (use -h for help)\n"))
+    1
+  else:
+    let (syncTargets, checkAurNames) = withAlpm(config.root, config.db,
+      config.dbs, config.arch, handle, dbs, errors):
+      for e in errors: printError(config.color, e)
+      findSyncTargets(handle, dbs, targets, false, false)
+
+    let (rpcInfos, aerrors) = getRpcPackageInfos(checkAurNames)
+    for e in aerrors: printError(config.color, e)
+
+    let notFoundTargets = filterNotFoundSyncTargets(syncTargets,
+      rpcInfos, initTable[string, PackageReference]())
+
+    if notFoundTargets.len > 0:
+      printSyncNotFound(config, notFoundTargets)
+      1
+    else:
+      let fullTargets = mapAurTargets[RpcPackageInfo](syncTargets, rpcInfos)
+      cloneAndCopy(config, quiet, fullTargets)