-
Notifications
You must be signed in to change notification settings - Fork 19
Expand file tree
/
Copy pathfiles.go
More file actions
998 lines (819 loc) · 30.8 KB
/
Copy pathfiles.go
File metadata and controls
998 lines (819 loc) · 30.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
package xtractr
/* Code to find, write, move and delete files. */
import (
"errors"
"fmt"
"io"
"maps"
"os"
"path/filepath"
"regexp"
"slices"
"strconv"
"strings"
"time"
"unicode/utf8"
)
// ArchiveList is the value returned when searching for compressed files.
// The map is directory to list of archives in that directory.
type ArchiveList map[string][]string
type archive struct {
Type string
// Ext is passed to strings.HasSuffix.
Ext string
// Fn is the extraction function for this extension.
Fn Interface
}
// Interface is a common interface for extracting compressed or non-compressed files or archives.
type Interface func(x *XFile) (size uint64, filesList, archiveList []string, err error)
// https://github.com/golift/xtractr/issues/44
//
// This list of archive types is used in a few places as extension lists.
//
//nolint:gochecknoglobals
var extension2function = []archive{
{Type: "tar.bzip2", Ext: ".tar.bz2", Fn: ChngInt(ExtractTarBzip)},
{Type: "cpio.gzip", Ext: ".cpio.gz", Fn: ChngInt(ExtractCPIOGzip)},
{Type: "tar.gzip", Ext: ".tar.gz", Fn: ChngInt(ExtractTarGzip)},
{Type: "tar.xz", Ext: ".tar.xz", Fn: ChngInt(ExtractTarXZ)},
{Type: "tar.lzw", Ext: ".tar.z", Fn: ChngInt(ExtractTarZ)},
// The ones with double extensions that match a single (below) need to come first.
{Type: "7zip", Ext: ".7z", Fn: Extract7z},
{Type: "7zip", Ext: ".7z.001", Fn: Extract7z},
{Type: "ar", Ext: ".ar", Fn: ChngInt(ExtractAr)},
{Type: "brotli", Ext: ".br", Fn: ChngInt(ExtractBrotli)},
{Type: "brotli", Ext: ".brotli", Fn: ChngInt(ExtractBrotli)},
{Type: "bz2", Ext: ".bz2", Fn: ChngInt(ExtractBzip)},
{Type: "cpio.gzip", Ext: ".cpgz", Fn: ChngInt(ExtractCPIOGzip)},
{Type: "cpio", Ext: ".cpio", Fn: ChngInt(ExtractCPIO)},
{Type: "deb", Ext: ".deb", Fn: ChngInt(ExtractAr)},
{Type: "gzip", Ext: ".gz", Fn: ChngInt(ExtractGzip)},
{Type: "gzip", Ext: ".gzip", Fn: ChngInt(ExtractGzip)},
{Type: "iso", Ext: ".iso", Fn: ChngInt(ExtractISO)},
{Type: "lz4", Ext: ".lz4", Fn: ChngInt(ExtractLZ4)},
{Type: "lzma", Ext: ".lz", Fn: ChngInt(ExtractLZMA)},
{Type: "lzma", Ext: ".lzip", Fn: ChngInt(ExtractLZMA)},
{Type: "lzma", Ext: ".lzma", Fn: ChngInt(ExtractLZMA)},
{Type: "lzma2", Ext: ".lzma2", Fn: ChngInt(ExtractLZMA2)},
{Type: "rar", Ext: ".r00", Fn: ExtractRAR},
{Type: "rar", Ext: ".rar", Fn: ExtractRAR},
{Type: "snappy2", Ext: ".s2", Fn: ChngInt(ExtractS2)},
{Type: "rpm", Ext: ".rpm", Fn: ChngInt(ExtractRPM)},
{Type: "snappy", Ext: ".snappy", Fn: ChngInt(ExtractSnappy)},
{Type: "snappy", Ext: ".sz", Fn: ChngInt(ExtractSnappy)},
{Type: "tar", Ext: ".tar", Fn: ChngInt(ExtractTar)},
{Type: "tar.bzip2", Ext: ".tbz", Fn: ChngInt(ExtractTarBzip)},
{Type: "tar.bzip2", Ext: ".tbz2", Fn: ChngInt(ExtractTarBzip)},
{Type: "tar.gzip", Ext: ".tgz", Fn: ChngInt(ExtractTarGzip)},
{Type: "tar.lzma", Ext: ".tlz", Fn: ChngInt(ExtractTarLzip)},
{Type: "tar.xz", Ext: ".txz", Fn: ChngInt(ExtractTarXZ)},
{Type: "tar.lzw", Ext: ".tz", Fn: ChngInt(ExtractTarZ)},
{Type: "xz", Ext: ".xz", Fn: ChngInt(ExtractXZ)},
{Type: "lzw", Ext: ".z", Fn: ChngInt(ExtractLZW)}, // everything is lowercase...
{Type: "zip", Ext: ".zip", Fn: ChngInt(ExtractZIP)},
{Type: "zlib", Ext: ".zlib", Fn: ChngInt(ExtractZlib)},
{Type: "zstandard", Ext: ".zst", Fn: ChngInt(ExtractZstandard)},
{Type: "zstandard", Ext: ".zstd", Fn: ChngInt(ExtractZstandard)},
{Type: "zlib", Ext: ".zz", Fn: ChngInt(ExtractZlib)},
{Type: "flac", Ext: ".cue", Fn: ExtractCUE},
}
// ChngInt converts the smaller return interface into an ExtractInterface.
// Functions with multi-part archive files return four values. Other functions return only 3.
// This ChngInt function makes both interfaces compatible.
func ChngInt(smallFn func(*XFile) (uint64, []string, error)) Interface {
return func(xFile *XFile) (uint64, []string, []string, error) {
size, files, err := smallFn(xFile)
return size, files, []string{xFile.FilePath}, err
}
}
// SupportedExtensions returns a slice of file extensions this library recognizes.
func SupportedExtensions() []string {
exts := make([]string, len(extension2function))
for idx, ext := range extension2function {
exts[idx] = ext.Ext
}
return exts
}
// XFile defines the data needed to extract an archive.
type XFile struct {
// Path to archive being extracted.
FilePath string
// Folder to extract archive into.
OutputDir string
// Write files with this mode.
FileMode os.FileMode
// Write folders with this mode.
DirMode os.FileMode
// (RAR/7z) Archive password. Blank for none. Gets prepended to Passwords, below.
Password string
// (RAR/7z) Archive passwords (to try multiple).
Passwords []string
// FileWorkers controls how many files within a single archive are extracted
// concurrently. Only effective for random-access formats (ZIP, 7z).
// Streaming formats ignore this. 0 or 1 = sequential (current behavior).
// Total concurrent I/O when using the queue = Config.Parallel * FileWorkers.
FileWorkers int
// Progress is called periodically during file extraction.
// Contains info about the progress of the extraction.
// This is not called if an Updates channel is also provided.
Progress func(Progress)
// If an Updates channel is provided, all Progress updates are sent to it.
// Contains info about the progress of the extraction.
Updates chan Progress
// If the archive only has one directory in the root, then setting
// this true will cause the extracted content to be moved into the
// output folder, and the root folder in the archive to be removed.
SquashRoot bool
// SkipOnRecursion, if set by an extractor, lists paths that were copied into
// the output (e.g. a CUE sheet) and must not be re-extracted when recursing.
SkipOnRecursion []string
// Logger allows printing debug messages.
log Logger
moveFiles func(fromPath, toPath string, overwrite bool) ([]string, error)
prog *progressTracker
}
// Filter is the input to find compressed files.
type Filter struct {
// This is the path to search in for archives.
Path string
// Any files with this suffix are ignored. ie. ".7z" or ".iso"
// Use the AllExcept func to create an inclusion list instead.
ExcludeSuffix Exclude
// Count of folder depth allowed when finding archives. 1 = root
MaxDepth int
// Only find archives this many child-folders deep. 0 and 1 are equal.
MinDepth int
}
// Exclude represents an exclusion list.
type Exclude []string
// Debugf calls the debug method on the logger if it's not nil.
func (x *XFile) Debugf(format string, v ...any) {
if x.log != nil {
x.log.Debugf(format, v...)
}
}
// Printf calls the print method on the logger if it's not nil.
func (x *XFile) Printf(format string, v ...any) {
if x.log != nil {
x.log.Printf(format, v...)
}
}
// GetFileList returns all the files in a path or paths.
// This is non-recursive and only returns files _in_ the base paths provided.
// This is a helper method and only exposed for convenience. You do not have to call this.
func (x *Xtractr) GetFileList(paths ...string) ([]string, error) {
files := []string{}
for _, path := range paths {
stat, err := os.Stat(path)
if err != nil {
return nil, fmt.Errorf("stat: %w", err)
}
if !stat.IsDir() {
files = append(files, path)
continue
}
fileList, err := os.ReadDir(path)
if err != nil {
return nil, fmt.Errorf("reading path %s: %w", path, err)
}
for _, file := range fileList {
files = append(files, filepath.Join(path, file.Name()))
}
}
return files, nil
}
// Difference returns all the strings that are in slice2 but not in slice1.
// Used to find new files in a file list from a path. ie. those we extracted.
// This is a helper method and only exposed for convenience. You do not have to call this.
func Difference(slice1, slice2 []string) []string {
diff := []string{}
for _, s2p := range slice2 {
var found bool
if slices.Contains(slice1, s2p) {
found = true
}
if !found { // String not found, so it's a new string, add it to the diff.
diff = append(diff, s2p)
}
}
return diff
}
// Has returns true if the test has an excluded suffix.
func (e Exclude) Has(test string) bool {
for _, exclude := range e {
if strings.HasSuffix(test, strings.ToLower(exclude)) {
return true
}
}
return false
}
// FindCompressedFiles returns all the compressed archive files in a path. This attempts to grab
// only the first file in a multi-part rar or 7zip archive. Sometimes there are multiple archives,
// so if the rar archive does not have "part" followed by a number in the name, then it will be
// considered an independent archive. Some packagers seem to use different naming schemes,
// so this may need to be updated as time progresses. Use the input to Filter to adjust the output.
func FindCompressedFiles(filter Filter) ArchiveList {
return findCompressedFiles(filter.Path, &filter, 0)
}
func findCompressedFiles(path string, filter *Filter, depth int) ArchiveList {
if filter.MaxDepth > 0 && filter.MaxDepth < depth {
return nil
}
dir, err := os.Open(path)
if err != nil {
return nil
}
defer dir.Close()
info, err := dir.Stat()
if err != nil {
return nil // unreadable folder?
}
if !info.IsDir() && IsArchiveFile(path) {
return ArchiveList{path: {path}} // passed in an archive file; send it back out.
}
fileList := getFilteredFileList(path, dir)
if len(fileList) == 0 {
return nil
}
return getCompressedFiles(path, filter, fileList, depth)
}
// getFilteredFileList reads the directory and returns a list of readable files that are not dot files.
func getFilteredFileList(path string, dir *os.File) []os.FileInfo {
names, _ := dir.Readdirnames(-1)
fileList := make([]os.FileInfo, 0, len(names))
for _, name := range names {
if name == "" || name[0] == '.' {
continue // skip dot files (including AppleDouble ._* entries)
}
info, err := os.Lstat(filepath.Join(path, name))
if err != nil {
continue // skip entries we can't stat
}
fileList = append(fileList, info)
}
return fileList
}
// IsArchiveFile returns true if the provided path has an archive file extension.
// This is not picky about extensions, and will match any that are known as an archive.
// In the future, it may use file magic to figure out if the file is an archive without
// relying on the extension.
func IsArchiveFile(path string) bool {
path = strings.ToLower(path)
for _, ext := range extension2function {
if strings.HasSuffix(path, ext.Ext) {
return true
}
}
return false
}
// CheckR00ForRarFile scans the file list to determine if a .rar file with the same name as .r00 exists.
// Returns true if the r00 files has an accompanying rar file in the fileList.
func CheckR00ForRarFile(fileList []os.FileInfo, r00file string) bool {
findFile := strings.TrimSuffix(strings.TrimSuffix(r00file, ".R00"), ".r00") + ".rar"
for _, file := range fileList {
if strings.EqualFold(file.Name(), findFile) {
return true
}
}
return false
}
// getCompressedFiles checks file suffixes to find archives to decompress.
// This pays special attention to the widely accepted variance of rar formats.
func getCompressedFiles(path string, filter *Filter, fileList []os.FileInfo, depth int) ArchiveList { //nolint:cyclop
files := ArchiveList{}
for _, file := range fileList {
switch lowerName := strings.ToLower(file.Name()); {
case !file.IsDir() &&
(filter.ExcludeSuffix.Has(lowerName) || depth < filter.MinDepth):
continue // file suffix is excluded or we are not deep enough.
case lowerName == "" || lowerName[0] == '.':
continue // ignore empty names and dot files/folders.
case file.IsDir(): // Recurse.
maps.Copy(files, findCompressedFiles(filepath.Join(path, file.Name()), filter, depth+1))
case strings.HasSuffix(lowerName, ".rar"):
hasParts := regexp.MustCompile(`.*\.part\d+\.rar$`)
partOne := regexp.MustCompile(`.*\.part0*1\.rar$`)
// Some archives are named poorly. Only return part01 or part001, not all.
if !hasParts.MatchString(lowerName) || partOne.MatchString(lowerName) {
files[path] = append(files[path], filepath.Join(path, file.Name()))
}
case strings.HasSuffix(lowerName, ".r00") && !CheckR00ForRarFile(fileList, lowerName):
// Accept .r00 as the first archive file if no .rar files are present in the path.
files[path] = append(files[path], filepath.Join(path, file.Name()))
case !strings.HasSuffix(lowerName, ".r00") && IsArchiveFile(lowerName):
files[path] = append(files[path], filepath.Join(path, file.Name()))
}
}
return files
}
// normalizeVolumes maps the volume list reported by an archive decoder into
// cleaned, deletable paths. Decoders are inconsistent: some return bare
// basenames (rardecode) while others return relative paths (sevenzip). Absolute
// paths and relative paths with directory components are preserved; bare names
// are resolved next to the entry archive file, where split archive volumes are
// expected to live. No filesystem probing is performed so the resulting cleanup
// paths are deterministic and independent of the process working directory.
// Empty or "." volume entries are dropped so cleanup never targets a directory;
// if no usable volumes remain, the entry file path is returned instead.
func normalizeVolumes(volumes []string, filePath string) []string {
filePath = filepath.Clean(filePath)
if len(volumes) == 0 {
return []string{filePath}
}
dir := filepath.Dir(filePath)
normalized := make([]string, 0, len(volumes))
for _, volume := range volumes {
volume = filepath.Clean(volume)
if volume == "." {
continue
}
if filepath.IsAbs(volume) || volume != filepath.Base(volume) {
normalized = append(normalized, volume)
continue
}
normalized = append(normalized, filepath.Join(dir, filepath.Base(volume)))
}
if len(normalized) == 0 {
return []string{filePath}
}
return normalized
}
// Extract calls the correct procedure for the type of file being extracted.
// Returns size of extracted data, list of extracted files, and/or error.
func (x *XFile) Extract() (size uint64, filesList, archiveList []string, err error) {
return ExtractFile(x)
}
// ExtractFile calls the correct procedure for the type of file being extracted.
// Returns size of extracted data, list of extracted files, list of archives processed, and/or error.
func ExtractFile(xFile *XFile) (size uint64, filesList, archiveList []string, err error) {
sName := strings.ToLower(xFile.FilePath)
// just borrowing this... Has to go into an interface to avoid a cycle.
xFile.moveFiles = parseConfig(&Config{Logger: xFile.log}).MoveFiles
var extensionType string // archive type from matched extension, for error reporting when extraction fails
for _, ext := range extension2function {
if strings.HasSuffix(sName, ext.Ext) {
size, filesList, archiveList, err = ext.Fn(xFile)
if err == nil {
return size, filesList, archiveList, nil
}
extensionType = ext.Type // preserve for error reporting before fallback
// Extension matched but extraction failed; try signature detection as fallback.
break
}
}
// Fall back to file signature (magic number) detection.
xFile.Debugf("falling back to signature detection for %s (extension error: %v)", xFile.FilePath, err)
extractFn, archiveType, sigErr := detectBySignature(xFile.FilePath)
if sigErr != nil {
extErr := &ExtractError{
FilePath: xFile.FilePath,
OutputDir: xFile.OutputDir,
ArchiveType: extensionType,
}
if err != nil {
extErr.Errs = append(extErr.Errs, err)
}
extErr.Errs = append(extErr.Errs, sigErr)
return 0, nil, nil, extErr
}
size, filesList, archiveList, err = extractFn(xFile)
if err != nil {
return size, filesList, archiveList, WrapExtractError(err, xFile, size, archiveType)
}
return size, filesList, archiveList, nil
}
// MoveFiles relocates files then removes the folder they were in.
// Returns the new file paths.
// This is a helper method and only exposed for convenience. You do not have to call this.
func (x *Xtractr) MoveFiles(fromPath, toPath string, overwrite bool) ([]string, error) { //nolint:cyclop
var (
newFiles = []string{}
keepErr error
)
files, err := x.GetFileList(fromPath)
if err != nil {
return nil, err
}
// If the "to path" is an existing archive file, remove the suffix to make a directory.
_, err = os.Stat(toPath)
if err == nil && IsArchiveFile(toPath) {
toPath = strings.TrimSuffix(toPath, filepath.Ext(toPath))
}
x.config.Debugf("Moving files: %v (%d files) -> %v", fromPath, len(files), toPath)
err = os.MkdirAll(toPath, x.config.DirMode)
if err != nil {
return nil, fmt.Errorf("making final dir: %w", err)
}
for _, file := range files {
var (
newFile = filepath.Join(toPath, filepath.Base(file))
_, err = os.Stat(newFile)
exists = !os.IsNotExist(err)
)
if exists && !overwrite {
x.config.Printf("Error: Renaming Temp File: %v to %v: (refusing to overwrite existing file)", file, newFile)
// keep trying.
continue
}
switch err = x.Rename(file, newFile); {
case err != nil:
keepErr = err
x.config.Printf("Error: Renaming Temp File: %v to %v: %v", file, newFile, err)
case exists:
newFiles = append(newFiles, newFile)
x.config.Debugf("Renamed Temp File: %v -> %v (overwrote existing file)", file, newFile)
default:
newFiles = append(newFiles, newFile)
x.config.Debugf("Renamed Temp File: %v -> %v", file, newFile)
}
}
x.DeleteFiles(fromPath)
// Since this is the last step, we tried to rename all the files, bubble the
// os.Rename error up, so it gets flagged as failed. It may have worked, but
// it should get attention.
return newFiles, keepErr
}
// DeleteFiles obliterates things and logs. Use with caution.
func (x *Xtractr) DeleteFiles(files ...string) {
for _, file := range files {
err := os.RemoveAll(file)
if err != nil {
x.config.Printf("Error: Deleting %v: %v", file, err)
continue
}
x.config.Printf("Deleted (recursively): %s", file)
}
}
// nameMax is the typical filesystem limit for a single path component (POSIX NAME_MAX).
const nameMax = 255
// TruncatePathForFS returns a path that fits within filesystem name limits by
// truncating the last path component (the filename) to nameMax bytes and, if
// that name already exists in the directory, appending ~1, ~2, etc. until an
// available name is found. The extension is preserved; the stem is truncated at
// UTF-8 rune boundaries. Use this when IsErrNameTooLong indicates a path is too long.
//
//nolint:nilerr
func TruncatePathForFS(path string) (string, error) {
var (
dir = filepath.Dir(path)
ext = filepath.Ext(path)
base = strings.TrimSuffix(filepath.Base(path), ext)
stem = truncateToBytes(base, max(nameMax-len(ext), 1))
tryPath = filepath.Join(dir, stem+ext)
)
_, err := os.Lstat(tryPath)
if err != nil { // path doesn't exist or other error; caller can try to create it
return tryPath, nil
}
for attempt := range 1000 {
postfix := "~" + strconv.Itoa(attempt+1)
newStem := truncateToBytes(stem, max(nameMax-len(ext)-len(postfix), 1))
tryPath = filepath.Join(dir, newStem+postfix+ext)
_, err = os.Lstat(tryPath)
if err != nil {
return tryPath, nil
}
}
return "", ErrNameTooLong
}
// truncateToBytes shortens s to at most maxBytes bytes, on UTF-8 rune boundaries.
// It returns s unchanged if maxBytes is negative or zero to avoid infinite loops or panics.
func truncateToBytes(str string, maxBytes int) string {
if maxBytes <= 0 || len(str) <= maxBytes {
if maxBytes <= 0 {
return ""
}
return str
}
raw := []byte(str)
for len(raw) > maxBytes {
_, size := utf8.DecodeLastRune(raw)
raw = raw[:len(raw)-size]
}
return string(raw)
}
// openFile opens path with the given flags and mode. If the path exceeds
// filesystem name limits, the path is truncated via TruncatePathForFS and
// retried. It returns the opened file and the path that was actually used
// (the original or the truncated path), so the caller can update file.Path
// for later use (e.g. os.Chtimes).
func openFile(path string, flags int, mode os.FileMode) (*os.File, string, error) {
openFile, err := os.OpenFile(path, flags, mode)
if err == nil {
return openFile, path, nil
}
if !IsErrNameTooLong(err) {
return nil, "", fmt.Errorf("os.OpenFile(): %w", err)
}
shortPath, truncErr := TruncatePathForFS(path)
if truncErr != nil {
return nil, "", truncErr
}
openFile, err = os.OpenFile(shortPath, flags, mode)
if err != nil {
return nil, "", fmt.Errorf("os.OpenFile(): %w", err)
}
return openFile, shortPath, nil
}
type file struct {
Path string
Data io.Reader
FileMode os.FileMode
DirMode os.FileMode
Mtime time.Time
Atime time.Time
// Linkname is an explicit symlink target when the archive format stores it
// outside the file payload (e.g. RAR5 redirection records).
Linkname string
}
// Rename is an attempt to deal with "invalid cross link device" on weird file systems.
func (x *Xtractr) Rename(oldpath, newpath string) error {
origErr := os.Rename(oldpath, newpath)
if origErr == nil {
return nil
}
origErr = fmt.Errorf("os.Rename(): %w", origErr)
/* Rename failed, try copy. */
oldFileStat, err := os.Stat(oldpath)
if err != nil {
return &ExtractError{Errs: []error{origErr, fmt.Errorf("os.Stat(): %w", err)}}
}
oldFile, err := os.Open(oldpath) // do not forget to close this!
if err != nil {
return &ExtractError{Errs: []error{origErr, fmt.Errorf("os.Open(): %w", err)}}
}
newFile, _, err := openFile(newpath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, oldFileStat.Mode())
if err != nil {
return &ExtractError{Errs: []error{origErr, err}}
}
defer newFile.Close()
_, err = io.Copy(newFile, oldFile)
if err != nil {
return &ExtractError{Errs: []error{origErr, fmt.Errorf("io.Copy(): %w", err)}}
}
_ = os.Chtimes(newpath, oldFileStat.ModTime(), oldFileStat.ModTime())
// The copy was successful, so now delete the original file
_ = oldFile.Close() // Needs to be closed before delete.
_ = os.Remove(oldpath)
return nil
}
// AllExcept can be used as an input to ExcludeSuffix in a Filter.
// Returns a list of supported extensions minus the ones provided.
// Extensions for like-types such as .rar and .r00 need to both be provided.
// Same for .tar.gz and .tgz variants.
func AllExcept(onlyThese ...string) Exclude {
// Start by excluding everything.
output := SupportedExtensions()
// Loop through the extensions we want to keep.
for _, str := range onlyThese {
idx := 0
// Remove each one from the output list.
for _, ext := range output {
if !strings.EqualFold(ext, str) {
output[idx] = ext
idx++
}
}
// Truncate the output to the size of items kept.
output = output[:idx]
}
return output
}
// Count returns the number of unique archives in the archive list.
func (a ArchiveList) Count() int {
var count int
for _, files := range a {
count += len(files)
}
return count
}
// Random returns a random file listing from the archive list.
// If the list only contains one directory, then that is the one returned.
// If the archive list is empty or nil, returns nil.
func (a ArchiveList) Random() []string {
for _, files := range a {
return files
}
return nil
}
// List returns all of the archives as a string slice.
func (a ArchiveList) List() []string {
list := make([]string, 0, len(a))
for _, files := range a {
list = append(list, files...)
}
return list
}
// SetLogger sets the logger interface on an XFile. Useful when you need to debug what it's doing.
func (x *XFile) SetLogger(logger Logger) {
x.log = logger
}
// cleanup runs after a successful extract.
// The intent it to move files into their final location.
func (x *XFile) cleanup(files []string) ([]string, error) {
files, err := x.squashRoot(files)
if err != nil {
return files, err
}
return files, nil
}
func (x *XFile) squashRoot(files []string) ([]string, error) {
if !x.SquashRoot {
return files, nil
}
roots := map[string]struct{}{}
for _, path := range files {
// Remove the output dir suffix, then split on `/` (or `\`) and get the first item.
newRoot := strings.TrimLeft(strings.TrimPrefix(path, x.OutputDir), string(filepath.Separator))
roots[strings.SplitN(newRoot, string(filepath.Separator), 2)[0]] = struct{}{} //nolint:mnd
}
if len(roots) == 1 { // only 1 root folder...
for root := range roots { // ...move it's content up a level.
return x.moveFiles(filepath.Join(x.OutputDir, root), x.OutputDir, false)
}
}
return files, nil
}
func (x *XFile) safeDirMode(current os.FileMode) os.FileMode {
if current.Perm() == 0 {
return x.DirMode
}
const minimum = 0o700 // ensure owner has read/write/exec on folders.
return current | minimum
}
func (x *XFile) safeFileMode(current os.FileMode) os.FileMode {
if current.Perm() == 0 {
return x.FileMode
}
const minimum = 0o400 // ensure owner has read access to the file.
return current | minimum
}
func openStatFile(path string) (*os.File, os.FileInfo, error) {
file, err := os.Open(path)
if err != nil {
return nil, nil, fmt.Errorf("os.Open: %w", err)
}
stat, err := file.Stat()
if err != nil {
_ = file.Close()
return nil, nil, fmt.Errorf("file.Stat: %w", err)
}
return file, stat, nil
}
func (x *XFile) mkDir(path string, mode os.FileMode, mtime time.Time) error {
defer os.Chtimes(path, time.Time{}, mtime)
return os.MkdirAll(path, x.safeDirMode(mode)) //nolint:wrapcheck
}
// write a file from an io reader, making sure all parent directories exist.
// Set parallel to true when writing from concurrent workers to throttle progress callbacks.
func (x *XFile) write(file *file) (uint64, error) {
return x.writeFile(file, false)
}
func (x *XFile) writeParallel(file *file) (uint64, error) {
return x.writeFile(file, true)
}
func (x *XFile) writeFile(file *file, parallel bool) (uint64, error) {
err := x.mkDir(filepath.Dir(file.Path), file.DirMode, file.Mtime)
if err != nil {
return 0, fmt.Errorf("writing archived file '%s' parent folder: %w", filepath.Base(file.Path), err)
}
// ZIP/RAR/7z (and similar) store symlink targets as the member payload with
// ModeSymlink set. Writing them as regular files leaves a text stub instead
// of a real link — the same class of bug as tar (#153), different symptom.
if file.FileMode&os.ModeSymlink != 0 {
err := x.writeSymlink(file)
if errors.Is(err, errSkipEntry) {
return 0, nil
}
return 0, err
}
fout, pathUsed, err := openFile(file.Path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, x.safeFileMode(file.FileMode))
if err != nil {
return 0, err
}
defer fout.Close()
file.Path = pathUsed
progWriter := x.prog.writer(fout)
if parallel {
progWriter = x.prog.parallelWriter(fout)
}
size, err := io.Copy(progWriter, file.Data)
if err != nil {
return uint64(size), fmt.Errorf("copying archived file '%s' io: %w", file.Path, err)
}
// The error is ignored because it's not critical and pops up on OSes like Windows.
defer os.Chtimes(file.Path, file.Atime, file.Mtime)
return uint64(size), nil
}
// maxSymlinkTarget is the maximum bytes allowed for a symlink target read from
// an archive member payload. Prevents a ModeSymlink entry with a huge payload
// from exhausting memory.
const maxSymlinkTarget = 8 * 1024
// writeSymlink reads a symlink target and creates the link at file.Path.
// Prefer file.Linkname when set (RAR5 redirections); otherwise read file.Data
// (ZIP/7z store the target as the member payload).
func (x *XFile) writeSymlink(file *file) error {
linkName := file.Linkname
if linkName == "" && file.Data != nil {
limited := io.LimitReader(file.Data, maxSymlinkTarget+1)
raw, err := io.ReadAll(limited)
if err != nil {
return fmt.Errorf("reading archived symlink '%s' target: %w", file.Path, err)
}
if len(raw) > maxSymlinkTarget {
return fmt.Errorf("%s: %w: %s", x.FilePath, ErrSymlinkTooLong, file.Path)
}
linkName = strings.TrimRight(string(raw), "\x00")
}
if linkName == "" {
x.Printf("Warning: skipping symlink with empty target: %s", file.Path)
return errSkipEntry
}
err := os.Remove(file.Path)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("%s: removing existing path for symlink: %w: %s", x.FilePath, err, file.Path)
}
return x.createSymlink(file.Path, linkName)
}
// clean returns an absolute path for a file inside the OutputDir.
// If trim length is > 0, then the suffixes are trimmed, and filepath removed.
func (x *XFile) clean(filePath string, trim ...string) string {
if len(trim) != 0 {
filePath = filepath.Base(filePath)
for _, suffix := range trim {
filePath = strings.TrimSuffix(filePath, suffix)
}
}
return filepath.Clean(filepath.Join(x.OutputDir, filePath))
}
// pathWithinOutput reports whether path is OutputDir or a descendant of it.
// Uses filepath.Rel so sibling-prefix tricks like OutputDir=/tmp/out and
// path=/tmp/out_evil fail (unlike strings.HasPrefix).
func (x *XFile) pathWithinOutput(path string) bool {
outputDir := filepath.Clean(x.OutputDir)
cleanPath := filepath.Clean(path)
rel, err := filepath.Rel(outputDir, cleanPath)
if err != nil {
return false
}
return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)))
}
// resolveLinkTarget returns the cleaned filesystem path a link would resolve to.
func resolveLinkTarget(linkPath, linkName string) string {
if filepath.IsAbs(linkName) {
return filepath.Clean(linkName)
}
return filepath.Clean(filepath.Join(filepath.Dir(linkPath), linkName))
}
// ensureLinkWithinOutput rejects symlink targets that escape OutputDir.
func (x *XFile) ensureLinkWithinOutput(linkPath, linkName string) error {
resolved := resolveLinkTarget(linkPath, linkName)
if !x.pathWithinOutput(resolved) {
return fmt.Errorf("%s: %w: %s (from: %s)", x.FilePath, ErrInvalidPath, resolved, linkName)
}
return nil
}
func (x *XFile) createSymlink(path, linkName string) error {
if linkName == "" {
x.Printf("Warning: skipping symlink with empty target: %s", path)
return errSkipEntry
}
err := x.ensureLinkWithinOutput(path, linkName)
if err != nil {
return err
}
x.Debugf("Writing archived symlink: %s -> %s", path, linkName)
err = os.Symlink(linkName, path)
if err != nil {
return fmt.Errorf("%s: creating symlink: %w: %s -> %s", x.FilePath, err, path, linkName)
}
return nil
}
func (x *XFile) createHardLink(path, linkName string) error {
if linkName == "" {
x.Printf("Warning: skipping hard link with empty target: %s", path)
return errSkipEntry
}
// Hard-link names are archive member paths, not arbitrary filesystem paths.
if filepath.IsAbs(linkName) {
return fmt.Errorf("%s: %w: %s", x.FilePath, ErrInvalidPath, linkName)
}
target := x.clean(linkName)
if !x.pathWithinOutput(target) {
return fmt.Errorf("%s: %w: %s (from: %s)", x.FilePath, ErrInvalidPath, target, linkName)
}
x.Debugf("Writing archived hard link: %s => %s", path, target)
err := os.Link(target, path)
if err == nil {
return nil
}
linkErr := err
rel, relErr := filepath.Rel(filepath.Dir(path), target)
if relErr != nil {
return fmt.Errorf("%s: creating hard link: %w: %s => %s", x.FilePath, linkErr, path, target)
}
// Fall back to a relative symlink when hard links are unavailable
// (e.g. target not extracted yet, or the filesystem does not support them).
x.Debugf("Hard link failed (%v); falling back to symlink: %s -> %s", linkErr, path, rel)
return x.createSymlink(path, rel)
}