Skip to content
This repository was archived by the owner on Sep 9, 2020. It is now read-only.

Commit

Permalink
internal/gps: update prune functions to use in-memory filesystemState
Browse files Browse the repository at this point in the history 
This commit update prune functions to use filesystemState instead of
repeated walks over the filesyste.

strip_vendor functions still walk the filesystem and will be updated in
a later commit.

Signed-off-by: Ibrahim AshShohail <[email protected]>
  • Loading branch information
@ibrasho
ibrasho committed Oct 6, 2017
1 parent c460889 commit 12a1b64
Show file tree
Hide file tree
Showing 6 changed files with 384 additions and 329 deletions.
2 changes: 1 addition & 1 deletion hack/lint.bash
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@ set -e
PKGS=$(go list ./... | grep -vF /vendor/)
go vet $PKGS
golint $PKGS
megacheck -unused.exported -ignore "github.com/golang/dep/internal/test/test.go:U1000 github.com/golang/dep/internal/gps/prune.go:U1000" $PKGS
megacheck -unused.exported -ignore "github.com/golang/dep/internal/test/test.go:U1000" $PKGS
254 changes: 111 additions & 143 deletions internal/gps/prune.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ type PruneOptions uint8

const (
// PruneNestedVendorDirs indicates if nested vendor directories should be pruned.
PruneNestedVendorDirs = 1 << iota
PruneNestedVendorDirs PruneOptions = 1 << iota
// PruneUnusedPackages indicates if unused Go packages should be pruned.
PruneUnusedPackages
// PruneNonGoFiles indicates if non-Go files should be pruned.
Expand Down Expand Up @@ -57,12 +57,13 @@ var (
// on the PruneOptions passed.
//
// A Lock must be passed if PruneUnusedPackages is toggled on.
func Prune(baseDir string, options PruneOptions, l Lock, logger *log.Logger) error {
func Prune(baseDir string, l Lock, options PruneOptions, logger *log.Logger) error {
// TODO(ibrasho) allow passing specific options per project

for _, lp := range l.Projects() {
projectDir := filepath.Join(baseDir, string(lp.Ident().ProjectRoot))
err := PruneProject(projectDir, lp, options, logger)
if err != nil {

if err := PruneProject(projectDir, lp, options, logger); err != nil {
return err
}
}
Expand All @@ -73,28 +74,31 @@ func Prune(baseDir string, options PruneOptions, l Lock, logger *log.Logger) err
// PruneProject remove excess files according to the options passed, from
// the lp directory in baseDir.
func PruneProject(baseDir string, lp LockedProject, options PruneOptions, logger *log.Logger) error {
projectDir := filepath.Join(baseDir, string(lp.Ident().ProjectRoot))
fs, err := deriveFilesystemState(baseDir)
if err != nil {
return errors.Wrap(err, "could not derive filesystem state")
}

if (options & PruneNestedVendorDirs) != 0 {
if err := pruneNestedVendorDirs(projectDir); err != nil {
if err := pruneNestedVendorDirs(baseDir); err != nil {
return err
}
}

if (options & PruneUnusedPackages) != 0 {
if err := pruneUnusedPackages(lp, projectDir, logger); err != nil {
if err := pruneUnusedPackages(lp, fs, logger); err != nil {
return errors.Wrap(err, "failed to prune unused packages")
}
}

if (options & PruneNonGoFiles) != 0 {
if err := pruneNonGoFiles(projectDir, logger); err != nil {
if err := pruneNonGoFiles(fs, logger); err != nil {
return errors.Wrap(err, "failed to prune non-Go files")
}
}

if (options & PruneGoTestFiles) != 0 {
if err := pruneGoTestFiles(projectDir, logger); err != nil {
if err := pruneGoTestFiles(fs, logger); err != nil {
return errors.Wrap(err, "failed to prune Go test files")
}
}
Expand All @@ -107,152 +111,141 @@ func pruneNestedVendorDirs(baseDir string) error {
return filepath.Walk(baseDir, stripVendor)
}

// pruneUnusedPackages deletes unimported packages found within baseDir.
// pruneVendorDirs deletes all nested vendor directories within baseDir.
// func pruneVendorDirs(fs filesystemState, logger *log.Logger) error {
// toDelete := collectNestedVendorDirs(fs)

// for _, path := range toDelete {
// logger.Printf(" * %s", path)

// if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
// return err
// }
// }

// return nil
// }

// func collectNestedVendorDirs(fs filesystemState) []string {
// toDelete := make([]string, 0, len(fs.dirs)/4)

// for _, dir := range fs.dirs {
// if filepath.Base(dir)
// toDelete = append(toDelete, dir)
// }

// for _, link := range fs.links {
// toDelete = append(toDelete)
// }

// return files
// }

// pruneUnusedPackages deletes unimported packages found in fs.
// Determining whether packages are imported or not is based on the passed LockedProject.
func pruneUnusedPackages(lp LockedProject, projectDir string, logger *log.Logger) error {
func pruneUnusedPackages(lp LockedProject, fs filesystemState, logger *log.Logger) error {
pr := string(lp.Ident().ProjectRoot)
logger.Printf("Calculating unused packages in %s to prune.\n", pr)
unusedPackages := calculateUnusedPackages(lp, fs)

unusedPackages, err := calculateUnusedPackages(lp, projectDir)
if err != nil {
return errors.Wrapf(err, "could not calculate unused packages in %s", pr)
}
logger.Printf("Found %d unused packages in %s:\n", len(unusedPackages), pr)

logger.Printf("Found the following unused packages in %s:\n", pr)
for pkg := range unusedPackages {
logger.Printf(" * %s\n", filepath.Join(pr, pkg))
}

unusedPackagesFiles, err := collectUnusedPackagesFiles(projectDir, unusedPackages)
if err != nil {
return errors.Wrapf(err, "could not collect unused packages' files in %s", pr)
}
toDelete := collectUnusedPackagesFiles(fs, unusedPackages)

logger.Printf("Deleting %d files in unused packages:", len(toDelete))

if err := deleteFiles(unusedPackagesFiles); err != nil {
return errors.Wrapf(err, "")
for _, path := range toDelete {
logger.Printf(" * %s\n", path)

if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return err
}
}

return nil
}

// calculateUnusedPackages generates a list of unused packages in lp.
func calculateUnusedPackages(lp LockedProject, projectDir string) (map[string]struct{}, error) {
// TODO(ibrasho): optimize this...
func calculateUnusedPackages(lp LockedProject, fs filesystemState) map[string]struct{} {
unused := make(map[string]struct{})
imported := make(map[string]struct{})

for _, pkg := range lp.Packages() {
imported[pkg] = struct{}{}
}

err := filepath.Walk(projectDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}

// Ignore anything that's not a directory.
if !info.IsDir() {
return nil
}
// Add the root package if it's not imported.
if _, ok := imported["."]; !ok {
unused["."] = struct{}{}
}

pkg, err := filepath.Rel(projectDir, path)
if err != nil {
return errors.Wrap(err, "unexpected error while calculating unused packages")
}
for _, dirPath := range fs.dirs {
pkg := filepath.ToSlash(dirPath)

pkg = filepath.ToSlash(pkg)
if _, ok := imported[pkg]; !ok {
unused[pkg] = struct{}{}
}
}

return nil
})

return unused, err
return unused
}

// collectUnusedPackagesFiles returns a slice of all files in the unused packages in projectDir.
func collectUnusedPackagesFiles(projectDir string, unusedPackages map[string]struct{}) ([]string, error) {
// collectUnusedPackagesFiles returns a slice of all files in the unused packages based on fs.
func collectUnusedPackagesFiles(fs filesystemState, unusedPackages map[string]struct{}) []string {
// TODO(ibrasho): is this useful?
files := make([]string, 0, len(unusedPackages))

err := filepath.Walk(projectDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
for _, path := range fs.files {
// Keep perserved files.
if isPreservedFile(filepath.Base(path)) {
continue
}

// Ignore directories.
if info.IsDir() {
return nil
}
pkg := filepath.ToSlash(filepath.Dir(path))

// Ignore perserved files.
if isPreservedFile(info.Name()) {
return nil
}

pkg, err := filepath.Rel(projectDir, filepath.Dir(path))
if err != nil {
return errors.Wrap(err, "unexpected error while calculating unused packages")
}

pkg = filepath.ToSlash(pkg)
if _, ok := unusedPackages[pkg]; ok {
files = append(files, path)
files = append(files, filepath.Join(fs.root, path))
}
}

return nil
})

return files, err
return files
}

// pruneNonGoFiles delete all non-Go files existing within baseDir.
// pruneNonGoFiles delete all non-Go files existing in fs.
// Files with names that are prefixed by any entry in preservedNonGoFiles
// are not deleted.
func pruneNonGoFiles(baseDir string, logger *log.Logger) error {
files, err := collectNonGoFiles(baseDir, logger)
if err != nil {
return errors.Wrap(err, "could not collect non-Go files")
}

if err := deleteFiles(files); err != nil {
return errors.Wrap(err, "could not prune Go test files")
}

return nil
}

// collectNonGoFiles returns a slice containing all non-Go files in baseDir.
// Files meeting the checks in isPreservedFile are not returned.
func collectNonGoFiles(baseDir string, logger *log.Logger) ([]string, error) {
files := make([]string, 0)

err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
func pruneNonGoFiles(fs filesystemState, logger *log.Logger) error {
// TODO(ibrasho) detemine a sane capacity
toDelete := make([]string, 0, len(fs.files)/4)

for _, path := range fs.files {
// Ignore Go files.
if strings.HasSuffix(path, ".go") {
continue
}

// Ignore directories.
if info.IsDir() {
return nil
// Ignore perserved files.
if isPreservedFile(filepath.Base(path)) {
continue
}

// Ignore all Go files.
if strings.HasSuffix(info.Name(), ".go") {
return nil
}
toDelete = append(toDelete, filepath.Join(fs.root, path))
}

// Ignore perserved files.
if isPreservedFile(info.Name()) {
return nil
}
logger.Printf("Deleting %d non-Go files:\n", len(toDelete))

files = append(files, path)
for _, path := range toDelete {
logger.Printf(" * %s\n", path)

return nil
})
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return err
}
}

return files, err
return nil
}

// isPreservedFile checks if the file name indicates that the file should be
Expand All @@ -275,51 +268,26 @@ func isPreservedFile(name string) bool {
return false
}

// pruneGoTestFiles deletes all Go test files (*_test.go) within baseDir.
func pruneGoTestFiles(baseDir string, logger *log.Logger) error {
files, err := collectGoTestFiles(baseDir)
if err != nil {
return errors.Wrap(err, "could not collect Go test files")
}

if err := deleteFiles(files); err != nil {
return errors.Wrap(err, "could not prune Go test files")
}

return nil
}

// collectGoTestFiles returns a slice contains all Go test files (any files
// prefixed with _test.go) in baseDir.
func collectGoTestFiles(baseDir string) ([]string, error) {
files := make([]string, 0)
// pruneGoTestFiles deletes all Go test files (*_test.go) in fs.
func pruneGoTestFiles(fs filesystemState, logger *log.Logger) error {
// TODO(ibrasho) detemine a sane capacity
toDelete := make([]string, 0, len(fs.files)/2)

err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}

// Ignore directories.
if info.IsDir() {
return nil
}

// Ignore any files that is not a Go test file.
if strings.HasSuffix(info.Name(), "_test.go") {
files = append(files, path)
for _, path := range fs.files {
if strings.HasSuffix(path, "_test.go") {
toDelete = append(toDelete, filepath.Join(fs.root, path))
}
}

return nil
})
logger.Printf("Deleting %d Go test files:\n", len(toDelete))

return files, err
}
for _, path := range toDelete {
logger.Printf(" * %s\n", path)

func deleteFiles(paths []string) error {
for _, path := range paths {
if err := os.Remove(path); err != nil {
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return err
}
}

return nil
}
Loading

0 comments on commit 12a1b64

Please sign in to comment.