From 5f7f4f2d713025f05cbc240323b6fe47bb6dbe24 Mon Sep 17 00:00:00 2001 From: kitsunyan Date: Mon, 30 Apr 2018 00:01:25 +0300 Subject: Add PKGBUILD retrieving --- src/feature/syncinfo.nim | 2 +- src/feature/syncinstall.nim | 61 ++++------------ src/feature/syncsource.nim | 173 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 49 deletions(-) create mode 100644 src/feature/syncsource.nim (limited to 'src/feature') diff --git a/src/feature/syncinfo.nim b/src/feature/syncinfo.nim index 42d7e91..b0765f4 100644 --- a/src/feature/syncinfo.nim +++ b/src/feature/syncinfo.nim @@ -101,7 +101,7 @@ proc handleTarget(config: Config, padding: int, args: seq[Argument], proc handleSyncInfo*(args: seq[Argument], config: Config): int = let (_, callArgs) = checkAndRefresh(config.color, args) - let targets = args.packageTargets + let targets = args.packageTargets(false) let (syncTargets, checkAurNames) = withAlpm(config.root, config.db, config.dbs, config.arch, handle, dbs, errors): diff --git a/src/feature/syncinstall.nim b/src/feature/syncinstall.nim index c214198..ab01df1 100644 --- a/src/feature/syncinstall.nim +++ b/src/feature/syncinstall.nim @@ -197,7 +197,7 @@ proc findDependencies(config: Config, handle: ptr AlpmHandle, dbs: seq[ptr AlpmD let (rpcInfos, aerrors) = getRpcPackageInfos(aurCheck.map(r => r.name)) for e in aerrors: printError(config.color, e) let (pkgInfos, additionalPkgInfos, paths, cerrors) = - cloneAurReposWithPackageInfos(config, rpcInfos, not printMode, update) + cloneAurReposWithPackageInfos(config, rpcInfos, not printMode, update, true) for e in cerrors: printError(config.color, e) (pkgInfos, additionalPkgInfos, paths)) @@ -247,30 +247,7 @@ proc findDependencies(config: Config, handle: ptr AlpmHandle, template clearPaths(paths: untyped) = for path in paths: removeDirQuiet(path) - discard rmdir(config.tmpRoot) - -proc filterNotFoundSyncTargetsInternal(syncTargets: seq[SyncPackageTarget], - pkgInfoReferencesTable: Table[string, PackageReference], - upToDateNeededTable: Table[string, PackageReference]): seq[SyncPackageTarget] = - # collect packages which were found neither in sync DB nor in AUR - syncTargets.filter(t => not (upToDateNeededTable.opt(t.reference.name) - .map(r => t.reference.isProvidedBy(r)).get(false)) and t.foundInfos.len == 0 and - not (t.isAurTargetSync and pkgInfoReferencesTable.opt(t.reference.name) - .map(r => t.reference.isProvidedBy(r)).get(false))) - -proc filterNotFoundSyncTargets[T: RpcPackageInfo](syncTargets: seq[SyncPackageTarget], - pkgInfos: seq[T], upToDateNeededTable: Table[string, PackageReference]): seq[SyncPackageTarget] = - let pkgInfoReferencesTable = pkgInfos.map(i => (i.name, i.toPackageReference)).toTable - filterNotFoundSyncTargetsInternal(syncTargets, pkgInfoReferencesTable, upToDateNeededTable) - -proc printSyncNotFound(config: Config, notFoundTargets: seq[SyncPackageTarget]) = - let dbs = config.dbs.toSet - - for target in notFoundTargets: - if target.repo.isNone or target.repo == some("aur") or target.repo.unsafeGet in dbs: - printError(config.color, trp("target not found: %s\n") % [$target.reference]) - else: - printError(config.color, trp("database not found: %s\n") % [target.repo.unsafeGet]) + discard rmdir(config.tmpRootInitial) proc printUnsatisfied(config: Config, satisfied: Table[PackageReference, SatisfyResult], unsatisfied: seq[PackageReference]) = @@ -328,19 +305,7 @@ proc editLoop(config: Config, base: string, repoPath: string, gitSubdir: Option[ else: res - let rawFiles = if gitSubdir.isSome: - forkWaitRedirect(() => (block: - dropPrivilegesAndChdir(none(string)): - execResult(gitCmd, "-C", repoPath, "ls-tree", "-r", "--name-only", "@", - gitSubdir.unsafeGet & "/"))) - .output - .map(s => s[gitSubdir.unsafeGet.len + 1 .. ^1]) - else: - forkWaitRedirect(() => (block: - dropPrivilegesAndChdir(none(string)): - execResult(gitCmd, "-C", repoPath, "ls-tree", "-r", "--name-only", "@"))) - .output - + let rawFiles = getGitFiles(repoPath, gitSubdir, true) let files = ("PKGBUILD" & rawFiles.filter(x => x != ".SRCINFO")).deduplicate proc editFileLoopAll(index: int): char = @@ -355,7 +320,7 @@ proc editLoop(config: Config, base: string, repoPath: string, gitSubdir: Option[ proc buildLoop(config: Config, pkgInfos: seq[PackageInfo], noconfirm: bool, noextract: bool): (Option[BuildResult], int, bool) = let base = pkgInfos[0].base - let repoPath = repoPath(config.tmpRoot, base) + let repoPath = repoPath(config.tmpRootInitial, base) let gitSubdir = pkgInfos[0].gitSubdir let buildPath = buildPath(repoPath, gitSubdir) @@ -365,7 +330,7 @@ proc buildLoop(config: Config, pkgInfos: seq[PackageInfo], noconfirm: bool, else: confFileEnv - let workConfFile = config.tmpRoot & "/makepkg.conf" + let workConfFile = config.tmpRootInitial & "/makepkg.conf" let workConfFileCopySuccess = try: copyFile(confFile, workConfFile) @@ -377,7 +342,7 @@ proc buildLoop(config: Config, pkgInfos: seq[PackageInfo], noconfirm: bool, file.writeLine("# PAKKU OVERRIDES") file.writeLine('#'.repeat(73)) file.writeLine("CARCH=" & config.arch.bashEscape) - file.writeLine("PKGDEST=" & config.tmpRoot.bashEscape) + file.writeLine("PKGDEST=" & config.tmpRootInitial.bashEscape) finally: file.close() true @@ -458,7 +423,7 @@ proc buildLoop(config: Config, pkgInfos: seq[PackageInfo], noconfirm: bool, proc buildFromSources(config: Config, commonArgs: seq[Argument], pkgInfos: seq[PackageInfo], noconfirm: bool): (Option[BuildResult], int) = let base = pkgInfos[0].base - let repoPath = repoPath(config.tmpRoot, base) + let repoPath = repoPath(config.tmpRootInitial, base) let gitSubdir = pkgInfos[0].gitSubdir proc loop(noextract: bool, showEditLoop: bool): (Option[BuildResult], int) = @@ -536,7 +501,7 @@ proc installGroupFromSources(config: Config, commonArgs: seq[Argument], proc formatArchiveFile(pkgInfo: PackageInfo, ext: string): string = let arch = if config.arch in pkgInfo.archs: config.arch else: "any" - config.tmpRoot & "/" & pkgInfo.name & "-" & pkgInfo.version & "-" & arch & ext + config.tmpRootInitial & "/" & pkgInfo.name & "-" & pkgInfo.version & "-" & arch & ext let allFiles = lc[(r.name, formatArchiveFile(r.pkgInfo, br.ext)) | (br <- buildResults, r <- br.replacePkgInfos), tuple[name: Option[string], file: string]] @@ -554,7 +519,7 @@ proc installGroupFromSources(config: Config, commonArgs: seq[Argument], discard if not clear: - printWarning(config.color, tr"packages are saved to '$#'" % [config.tmpRoot]) + printWarning(config.color, tr"packages are saved to '$#'" % [config.tmpRootInitial]) if buildCode != 0: handleTmpRoot(true) @@ -654,7 +619,7 @@ proc confirmViewAndImportKeys(config: Config, basePackages: seq[seq[seq[PackageI if index < flatBasePackages.len: let pkgInfos = flatBasePackages[index] let base = pkgInfos[0].base - let repoPath = repoPath(config.tmpRoot, base) + let repoPath = repoPath(config.tmpRootInitial, base) let aur = pkgInfos[0].repo == "aur" @@ -1072,7 +1037,7 @@ proc obtainAurPackageInfos(config: Config, rpcInfos: seq[RpcPackageInfo], else: (block: let (rpcInfos, aerrors) = getRpcPackageInfos(fullRpcInfos.map(i => i.name)) let (pkgInfos, additionalPkgInfos, paths, cerrors) = - cloneAurReposWithPackageInfos(config, rpcInfos, not printMode, update) + cloneAurReposWithPackageInfos(config, rpcInfos, not printMode, update, true) (pkgInfos, additionalPkgInfos, paths, (toSeq(aerrors.items) & cerrors).deduplicate)) terminate() @@ -1113,7 +1078,7 @@ proc obtainPacmanBuildTargets(config: Config, pacmanTargets: seq[FullPackageTarg let (buildPkgInfos, buildPaths, obtainErrorMessages) = if checkPacmanBuildPkgInfos: (block: echo(tr"checking official repositories...") let (update, terminate) = createCloneProgress(config, pacmanTargets.len, printMode) - let res = obtainBuildPkgInfos[PackageInfo](config, pacmanTargets, update) + let res = obtainBuildPkgInfos[PackageInfo](config, pacmanTargets, update, true) terminate() res) else: @@ -1259,7 +1224,7 @@ proc handleSyncInstall*(args: seq[Argument], config: Config): int = arg.matchOption(%%%"noconfirm")).optLast .map(arg => arg.key == "noconfirm").get(false) - let targets = args.packageTargets + let targets = args.packageTargets(false) withAur(): let (code, installed, foreignUpgrade, targetNamesSet, pacmanTargets, diff --git a/src/feature/syncsource.nim b/src/feature/syncsource.nim new file mode 100644 index 0000000..fba7062 --- /dev/null +++ b/src/feature/syncsource.nim @@ -0,0 +1,173 @@ +import + future, options, os, posix, sequtils, strutils, tables, + "../args", "../aur", "../common", "../config", "../format", "../lists", + "../package", "../pacman", "../utils", + "../wrapper/alpm" + +type + BaseTarget = tuple[ + base: string, + version: string, + destination: string, + aurGitUrl: Option[string], + gitRepo: Option[GitRepo] + ] + + CloneResult = tuple[ + base: string, + path: string, + files: seq[string], + destination: string + ] + +proc getFilesOrClear(base: string, repoPath: string, gitSubdir: Option[string]): + (seq[string], Option[string]) = + let rawFiles = getGitFiles(repoPath, gitSubdir, false) + .filter(f => f != ".gitignore" and f != ".SRCINFO" and f.find('/') < 0) + .map(f => gitSubdir.map(s => s & "/" & f).get(f)) + + if rawFiles.len > 0: + (rawFiles, none(string)) + else: + removeDirQuiet(repoPath) + (newSeq[string](), some(tr"$#: failed to clone git repository" % [base])) + +proc cloneRepositories(config: Config, targets: seq[BaseTarget], + update: (int, int) -> void): (List[CloneResult], List[string]) = + proc cloneNext(index: int, results: List[CloneResult], messages: List[string]): + (List[CloneResult], List[string]) = + update(index, targets.len) + + if index >= targets.len: + (results.reversed, messages.reversed) + else: + let target = targets[index] + let repoPath = repoPath(config.tmpRootCurrent, target.base) + removeDirQuiet(repoPath) + + if target.aurGitUrl.isSome: + let (cloneCode, cerror) = cloneAurRepo(config, + target.base, target.aurGitUrl.unsafeGet, false) + + if cloneCode != 0: + cloneNext(index + 1, results, toSeq(cerror.items) ^& messages) + else: + let (files, ferror) = getFilesOrClear(target.base, repoPath, none(string)) + if ferror.isSome: + cloneNext(index + 1, results, ferror.unsafeGet ^& messages) + else: + cloneNext(index + 1, (target.base, repoPath, files, + target.destination) ^& results, messages) + elif target.gitRepo.isSome: + let gitRepo = target.gitRepo.unsafeGet + let cerror = clonePackageRepo(config, target.base, + target.version, gitRepo, false) + + if cerror.isSome: + cloneNext(index + 1, results, cerror.unsafeGet ^& messages) + else: + let (files, ferror) = getFilesOrClear(target.base, repoPath, some(gitRepo.path)) + if ferror.isSome: + cloneNext(index + 1, results, ferror.unsafeGet ^& messages) + else: + cloneNext(index + 1, (target.base, repoPath, files, + target.destination) ^& results, messages) + else: + let message = tr"$#: repository not found" % [target.base] + cloneNext(index + 1, results, message ^& messages) + + cloneNext(0, nil, nil) + +proc copyFiles(config: Config, quiet: bool, results: seq[CloneResult]): List[string] = + proc copyNext(index: int, messages: List[string]): List[string] = + if index >= results.len: + messages.reversed + else: + let res = results[index] + discard mkdir(res.destination, 0o755) + + let error = try: + for f in res.files: + let index = f.rfind('/') + let name = if index >= 0: f[index + 1 .. ^1] else: f + let dest = if res.destination == ".": name else: res.destination & "/" & name + moveFile(res.path & "/" & f, dest) + printFile(config.color, quiet, res.base, dest) + none(string) + except OSError: + some(tr"$#: failed to move files" % [res.base]) + + copyNext(index + 1, toSeq(error.items) ^& messages) + + copyNext(0, nil) + +proc cloneAndCopy(config: Config, quiet: bool, + fullTargets: seq[FullPackageTarget[RpcPackageInfo]]): int = + let baseTargets = fullTargets.foldl(block: + let bases = a.map(x => x.base) + if b.isAurTargetFull: + let rpcInfo = b.pkgInfo.get + if rpcInfo.base in bases: + a + else: + a & (rpcInfo.base, rpcInfo.version, b.destination.get(rpcInfo.base), + some(rpcInfo.gitUrl), none(GitRepo)) + else: + let foundInfo = b.foundInfos[0] + let pkg = foundInfo.pkg.get + if pkg.base in bases: + a + else: + let git = lookupGitRepo(foundInfo.repo, pkg.base, pkg.arch.get) + a & (pkg.base, pkg.version, b.destination.get(pkg.base), + none(string), git), + newSeq[BaseTarget]()) + + let (update, terminate) = if quiet: + (proc (a: int, b: int) {.closure.} = discard, proc () {.closure.} = discard) + else: + printProgressShare(config.progressBar, tr"cloning repositories") + + let (results, rerrors) = cloneRepositories(config, baseTargets, update) + terminate() + for e in rerrors: printError(config.color, e) + + let cerrors = copyFiles(config, quiet, toSeq(results.items)) + for e in cerrors: printError(config.color, e) + + for result in results: + removeDirQuiet(result.path) + discard rmdir(config.tmpRootCurrent) + + if rerrors != nil and cerrors != nil: + 1 + else: + 0 + +proc handleSyncSource*(args: seq[Argument], config: Config): int = + discard checkAndRefresh(config.color, args) + + let quiet = args.check(%%%"quiet") + let targets = args.packageTargets(true) + + if targets.len == 0: + printError(config.color, trp("no targets specified (use -h for help)\n")) + 1 + else: + let (syncTargets, checkAurNames) = withAlpm(config.root, config.db, + config.dbs, config.arch, handle, dbs, errors): + for e in errors: printError(config.color, e) + findSyncTargets(handle, dbs, targets, false, false) + + let (rpcInfos, aerrors) = getRpcPackageInfos(checkAurNames) + for e in aerrors: printError(config.color, e) + + let notFoundTargets = filterNotFoundSyncTargets(syncTargets, + rpcInfos, initTable[string, PackageReference]()) + + if notFoundTargets.len > 0: + printSyncNotFound(config, notFoundTargets) + 1 + else: + let fullTargets = mapAurTargets[RpcPackageInfo](syncTargets, rpcInfos) + cloneAndCopy(config, quiet, fullTargets) -- cgit v1.2.3-70-g09d2