diff options
| -rw-r--r-- | src/common.nim | 155 | ||||
| -rw-r--r-- | src/feature/syncinstall.nim | 19 | ||||
| -rw-r--r-- | src/feature/syncsource.nim | 18 | ||||
| -rw-r--r-- | src/package.nim | 33 |
4 files changed, 158 insertions, 67 deletions
diff --git a/src/common.nim b/src/common.nim index 351dd73..b00b482 100644 --- a/src/common.nim +++ b/src/common.nim @@ -33,11 +33,6 @@ type repo: string ] - LookupGitResult = tuple[ - group: LookupBaseGroup, - git: Option[GitRepo] - ] - proc checkAndRefresh*(color: bool, args: seq[Argument]): tuple[code: int, args: seq[Argument]] = let refreshCount = args.count(%%%"refresh") if refreshCount > 0: @@ -419,17 +414,70 @@ proc reloadPkgInfos*(config: Config, path: string, pkgInfos: seq[PackageInfo]): else: pkgInfos +proc cloneBareRepo(config: Config, bareName: string, url: string, + dropPrivileges: bool): Option[string] = + let fullName = bareName & ".git" + let repoPath = repoPath(config.tmpRoot(dropPrivileges), fullName) + removeDirQuiet(repoPath) + + if forkWait(() => (block: + if not dropPrivileges or dropPrivileges(): + execResult(gitCmd, "-C", config.tmpRoot(dropPrivileges), + "clone", "-q", "--bare", url, fullName) + else: + quit(1))) == 0: + some(repoPath) + else: + removeDirQuiet(repoPath) + none(string) + +proc cloneBareRepos*(config: Config, gitRepos: seq[GitRepo], + progressCallback: (int, int) -> void, dropPrivileges: bool): (seq[string], seq[string]) = + let message = ensureTmpOrError(config, dropPrivileges) + if message.isSome: + (@[], @[message.unsafeGet]) + else: + let bare = gitRepos + .filter(t => t.bareName.isSome) + .map(r => (r.bareName.unsafeGet, r.url)) + .deduplicate + + proc cloneNext(index: int, paths: List[string], messages: List[string]): + (List[string], List[string]) = + progressCallback(index, bare.len) + + if index >= bare.len: + (paths.reversed, messages.reversed) + else: + let (bareName, url) = bare[index] + let repoPath = cloneBareRepo(config, bareName, url, dropPrivileges) + if repoPath.isSome: + cloneNext(index + 1, repoPath.unsafeGet ^& paths, messages) + else: + let message = tr"$#: failed to clone git repository" % [bareName] + cloneNext(index + 1, paths, message ^& messages) + + let (paths, messages) = cloneNext(0, nil, nil) + (toSeq(paths.items), toSeq(messages.items)) + proc clonePackageRepoInternal(config: Config, base: string, version: string, git: GitRepo, dropPrivileges: bool): Option[string] = - let tmpRoot = config.tmpRoot(dropPrivileges) - let repoPath = repoPath(tmpRoot, base) + let repoPath = repoPath(config.tmpRoot(dropPrivileges), base) removeDirQuiet(repoPath) + let url = if git.bareName.isSome: + repoPath(config.tmpRoot(dropPrivileges), git.bareName.unsafeGet & ".git") + else: + git.url + if forkWait(() => (block: if not dropPrivileges or dropPrivileges(): - execResult(gitCmd, "-C", tmpRoot, - "clone", "-q", git.url, "-b", git.branch, - "--single-branch", base) + if git.branch.isSome: + execResult(gitCmd, "-C", config.tmpRoot(dropPrivileges), + "clone", "-q", url, "-b", git.branch.unsafeGet, "--single-branch", base) + else: + execResult(gitCmd, "-C", config.tmpRoot(dropPrivileges), + "clone", "-q", url, "--single-branch", base) else: quit(1))) == 0: let commit = bisectVersion(repoPath, config.debug, none(string), @@ -466,7 +514,7 @@ proc clonePackageRepo*(config: Config, base: string, version: string, proc obtainBuildPkgInfosInternal(config: Config, bases: seq[LookupBaseGroup], pacmanTargetNames: seq[string], progressCallback: (int, int) -> void, dropPrivileges: bool): (seq[PackageInfo], seq[string], seq[string]) = - let lookupResults: seq[LookupGitResult] = bases + let lookupResults: seq[tuple[group: LookupBaseGroup, git: Option[GitRepo]]] = bases .map(b => (b, lookupGitRepo(b.repo, b.base, b.arch))) let notFoundRepos = lookupResults.filter(r => r.git.isNone) @@ -478,45 +526,56 @@ proc obtainBuildPkgInfosInternal(config: Config, bases: seq[LookupBaseGroup], if message.isSome: (@[], @[], @[message.unsafeGet]) else: - proc findCommitAndGetSrcInfo(base: string, version: string, - repo: string, git: GitRepo): tuple[pkgInfos: seq[PackageInfo], path: Option[string]] = - let repoPath = clonePackageRepoInternal(config, base, version, git, dropPrivileges) + let (barePaths, berrors) = cloneBareRepos(config, lookupResults.map(r => r.git.unsafeGet), + proc (progress: int, count: int) = progressCallback(progress, count + lookupResults.len), + dropPrivileges) - if repoPath.isSome: - let srcInfo = obtainSrcInfo(repoPath.unsafeGet & "/" & git.path) - let pkgInfos = parseSrcInfo(repo, srcInfo, config.arch, - git.url, some(git.path)) - .filter(i => i.version == version) - (pkgInfos, repoPath) - else: - (newSeq[PackageInfo](), none(string)) - - progressCallback(0, lookupResults.len) - let (pkgInfosWithPathsReversed, _) = lookupResults.foldl(block: - let (list, index) = a - let res = findCommitAndGetSrcInfo(b.group.base, b.group.version, - b.group.repo, b.git.get) ^& list - progressCallback(index + 1, lookupResults.len) - (res, index + 1), - (list[tuple[pkgInfos: seq[PackageInfo], path: Option[string]]](), 0)) - - let pkgInfosWithPaths = pkgInfosWithPathsReversed.reversed - let pkgInfos = lc[x | (y <- pkgInfosWithPaths, x <- y.pkgInfos), PackageInfo] - let paths = lc[x | (y <- pkgInfosWithPaths, x <- y.path), string] - - let pkgInfosTable = pkgInfos.map(i => (i.name, i)).toTable - - let foundPkgInfos = lc[x | (y <- pacmanTargetNames, - x <- pkgInfosTable.opt(y)), PackageInfo] - let errorMessages = pacmanTargetNames - .filter(n => not pkgInfosTable.hasKey(n)) - .map(n => tr"$#: failed to get package info" % [n]) - - if errorMessages.len > 0: - for path in paths: + if berrors.len > 0: + for path in barePaths: removeDirQuiet(path) - discard rmdir(config.tmpRoot(dropPrivileges)) - (foundPkgInfos, paths, errorMessages) + discard rmdir(config.tmpRoot(dropPrivileges)) + (newSeq[PackageInfo](), barePaths, berrors) + else: + proc findCommitAndGetSrcInfo(base: string, version: string, + repo: string, git: GitRepo): tuple[pkgInfos: seq[PackageInfo], path: Option[string]] = + let repoPath = clonePackageRepoInternal(config, base, version, git, dropPrivileges) + + if repoPath.isSome: + let srcInfo = obtainSrcInfo(repoPath.unsafeGet & "/" & git.path) + let pkgInfos = parseSrcInfo(repo, srcInfo, config.arch, + git.url, some(git.path)) + .filter(i => i.version == version) + (pkgInfos, repoPath) + else: + (newSeq[PackageInfo](), none(string)) + + let (pkgInfosWithPathsReversed, _) = lookupResults.foldl(block: + let (list, index) = a + let res = findCommitAndGetSrcInfo(b.group.base, b.group.version, + b.group.repo, b.git.unsafeGet) ^& list + progressCallback(barePaths.len + index + 1, barePaths.len + lookupResults.len) + (res, index + 1), + (list[tuple[pkgInfos: seq[PackageInfo], path: Option[string]]](), 0)) + + let pkgInfosWithPaths = pkgInfosWithPathsReversed.reversed + let pkgInfos = lc[x | (y <- pkgInfosWithPaths, x <- y.pkgInfos), PackageInfo] + let paths = lc[x | (y <- pkgInfosWithPaths, x <- y.path), string] + + let pkgInfosTable = pkgInfos.map(i => (i.name, i)).toTable + + let foundPkgInfos = lc[x | (y <- pacmanTargetNames, + x <- pkgInfosTable.opt(y)), PackageInfo] + let errorMessages = pacmanTargetNames + .filter(n => not pkgInfosTable.hasKey(n)) + .map(n => tr"$#: failed to get package info" % [n]) + + if errorMessages.len > 0: + for path in barePaths: + removeDirQuiet(path) + for path in paths: + removeDirQuiet(path) + discard rmdir(config.tmpRoot(dropPrivileges)) + (foundPkgInfos, barePaths & paths, errorMessages) proc obtainBuildPkgInfos*[T: RpcPackageInfo](config: Config, pacmanTargets: seq[FullPackageTarget[T]], progressCallback: (int, int) -> void, diff --git a/src/feature/syncinstall.nim b/src/feature/syncinstall.nim index ab01df1..61bd857 100644 --- a/src/feature/syncinstall.nim +++ b/src/feature/syncinstall.nim @@ -37,17 +37,20 @@ type proc groupsSeq(pkg: ptr AlpmPackage): seq[string] = toSeq(pkg.groups.items).map(s => $s) -proc createCloneProgress(config: Config, count: int, printMode: bool): +proc createCloneProgress(config: Config, count: int, flexible: bool, printMode: bool): (proc (update: int, terminate: int) {.closure.}, proc {.closure.}) = if count >= 1 and not printMode: let (update, terminate) = printProgressShare(config.progressBar, tr"cloning repositories") update(0, count) - proc cloneUpdate(progress: int, newCount: int) {.closure.} = - # newCount can be < count if some packages were not found - update(max(count - newCount + progress, 0), count) + if flexible: + proc cloneUpdate(progress: int, newCount: int) {.closure.} = + # newCount can be < count if some packages were not found + update(max(count - newCount + progress, 0), count) - (cloneUpdate, terminate) + (cloneUpdate, terminate) + else: + (update, terminate) else: (proc (a: int, b: int) {.closure.} = discard, proc {.closure.} = discard) @@ -185,7 +188,7 @@ proc findDependencies(config: Config, handle: ptr AlpmHandle, dbs: seq[ptr AlpmD let (aurSuccess, aurFail, newPaths, newAdditionalPkgInfos) = if not noaur and aurCheck.len > 0: (block: - let (update, terminate) = createCloneProgress(config, aurCheck.len, printMode) + let (update, terminate) = createCloneProgress(config, aurCheck.len, true, printMode) try: withAur(): let (pkgInfos, additionalPkgInfos, paths) = if printMode: (block: @@ -1028,7 +1031,7 @@ proc obtainAurPackageInfos(config: Config, rpcInfos: seq[RpcPackageInfo], let upgradeRpcInfos = upgradeStructs.filter(p => p.needed).map(p => p.rpcInfo) let fullRpcInfos = targetRpcInfos & upgradeRpcInfos - let (update, terminate) = createCloneProgress(config, fullRpcInfos.len, printMode) + let (update, terminate) = createCloneProgress(config, fullRpcInfos.len, true, printMode) let (pkgInfos, additionalPkgInfos, paths, errors) = if printMode: (block: let (pkgInfos, additionalPkgInfos, aerrors) = @@ -1077,7 +1080,7 @@ proc obtainPacmanBuildTargets(config: Config, pacmanTargets: seq[FullPackageTarg let (buildPkgInfos, buildPaths, obtainErrorMessages) = if checkPacmanBuildPkgInfos: (block: echo(tr"checking official repositories...") - let (update, terminate) = createCloneProgress(config, pacmanTargets.len, printMode) + let (update, terminate) = createCloneProgress(config, pacmanTargets.len, false, printMode) let res = obtainBuildPkgInfos[PackageInfo](config, pacmanTargets, update, true) terminate() res) diff --git a/src/feature/syncsource.nim b/src/feature/syncsource.nim index fba7062..f7e2524 100644 --- a/src/feature/syncsource.nim +++ b/src/feature/syncsource.nim @@ -33,10 +33,14 @@ proc getFilesOrClear(base: string, repoPath: string, gitSubdir: Option[string]): (newSeq[string](), some(tr"$#: failed to clone git repository" % [base])) proc cloneRepositories(config: Config, targets: seq[BaseTarget], - update: (int, int) -> void): (List[CloneResult], List[string]) = + update: (int, int) -> void): (List[CloneResult], seq[string], seq[string]) = + let (barePaths, berrors) = cloneBareRepos(config, + targets.filter(t => t.gitRepo.isSome).map(t => t.gitRepo.unsafeGet), + proc (progress: int, count: int) = update(progress, count + targets.len), false) + proc cloneNext(index: int, results: List[CloneResult], messages: List[string]): (List[CloneResult], List[string]) = - update(index, targets.len) + update(barePaths.len + index, barePaths.len + targets.len) if index >= targets.len: (results.reversed, messages.reversed) @@ -76,7 +80,11 @@ proc cloneRepositories(config: Config, targets: seq[BaseTarget], let message = tr"$#: repository not found" % [target.base] cloneNext(index + 1, results, message ^& messages) - cloneNext(0, nil, nil) + if berrors.len > 0: + (nil, barePaths, berrors) + else: + let (results, cerrors) = cloneNext(0, nil, nil) + (results, barePaths, toSeq(cerrors.items)) proc copyFiles(config: Config, quiet: bool, results: seq[CloneResult]): List[string] = proc copyNext(index: int, messages: List[string]): List[string] = @@ -128,7 +136,7 @@ proc cloneAndCopy(config: Config, quiet: bool, else: printProgressShare(config.progressBar, tr"cloning repositories") - let (results, rerrors) = cloneRepositories(config, baseTargets, update) + let (results, barePaths, rerrors) = cloneRepositories(config, baseTargets, update) terminate() for e in rerrors: printError(config.color, e) @@ -137,6 +145,8 @@ proc cloneAndCopy(config: Config, quiet: bool, for result in results: removeDirQuiet(result.path) + for path in barePaths: + removeDirQuiet(path) discard rmdir(config.tmpRootCurrent) if rerrors != nil and cerrors != nil: diff --git a/src/package.nim b/src/package.nim index 3884a34..c9ea5b2 100644 --- a/src/package.nim +++ b/src/package.nim @@ -53,7 +53,8 @@ type GitRepo* = tuple[ url: string, - branch: string, + bareName: Option[string], + branch: Option[string], path: string ] @@ -69,12 +70,12 @@ const packageRepos: seq[PackageRepo] = @[ (["arch"].toSet, ["core", "extra", "testing"].toSet, - ("https://git.archlinux.org/svntogit/packages.git", - "packages/${BASE}", "repos/${REPO}-${ARCH}")), + ("https://git.archlinux.org/svntogit/packages.git", none(string), + some("packages/${BASE}"), "repos/${REPO}-${ARCH}")), (["arch"].toSet, ["community", "community-testing", "multilib", "multilib-testing"].toSet, - ("https://git.archlinux.org/svntogit/community.git", - "packages/${BASE}", "repos/${REPO}-${ARCH}")) + ("https://git.archlinux.org/svntogit/community.git", none(string), + some("packages/${BASE}"), "repos/${REPO}-${ARCH}")) ] static: @@ -89,6 +90,23 @@ static: raise newException(SystemError, "only single matching repo available: " & os & ":" & repo) + # test unique url <> bareName links + let bareNameToUrl = lc[(x, r.git.url) | + (r <- packageRepos, x <- r.git.bareName), (string, string)].toTable + let urlToBareName = lc[(r.git.url, x) | + (r <- packageRepos, x <- r.git.bareName), (string, string)].toTable + + template testBareNamesAndUrls(m1: untyped, m2: untyped) = + for x1, x2 in m1: + try: + if m2[x2] != x1: + raise newException(SystemError, "") + except: + raise newException(SystemError, "Invalid url <> bareName links") + + testBareNamesAndUrls(bareNameToUrl, urlToBareName) + testBareNamesAndUrls(urlToBareName, bareNameToUrl) + proc readOsId: Option[string] = var file: File if file.open("/usr/lib/os-release"): @@ -108,7 +126,7 @@ proc readOsId: Option[string] = let osId = readOsId() proc lookupGitRepo*(repo: string, base: string, arch: string): Option[GitRepo] = - template replaceAll(gitPart: string): string = + proc replaceAll(gitPart: string): string = gitPart .replace("${REPO}", repo) .replace("${BASE}", base) @@ -116,7 +134,8 @@ proc lookupGitRepo*(repo: string, base: string, arch: string): Option[GitRepo] = packageRepos .filter(pr => osId.isSome and osid.unsafeGet in pr.os and repo in pr.repo) - .map(pr => (pr.git.url.replaceAll, pr.git.branch.replaceAll, pr.git.path.replaceAll)) + .map(pr => (pr.git.url.replaceAll, pr.git.bareName, + pr.git.branch.map(replaceAll), pr.git.path.replaceAll)) .optFirst template repoPath*(tmpRoot: string, base: string): string = |