pathlibs were being written!

What!  I thought we fixed this -- and now they're not safe across Mac to Unix!

Author: mscuthbert

dist/dist.py

@@ -79,7 +79,7 @@
 
     Finish this before doing the next step, even though it looks like it could be done in parallel.
 
-19a. Upload the new file to PyPI with "twine upload music21-9.3.0.tar.gz" [*]
+19a. Upload the tar.gz file to PyPI with "twine upload music21-9.3.0.tar.gz" [*]
 
 19b. Do the same for the whl file (but not for the no-corpus file) [*]
 
@@ -95,6 +95,10 @@
         username:__token__
         password:pypi-the_gibberish_generated_in_create_api_token_at_the_bottom_of_account_settings
 
+    The "password" is under "Account Settings" -> "API Token" -> "Options" -> "View Unique
+    Identifier" -- the Copy button doesn't work at least on Mac.
+    (or you can use the "password" field in the [pypi] section, but that's not recommended)')
+
     FYI -- PyPI is apparently uninterested in having contributions from smaller projects
     which haven't been following their internal security discussions for years.  All of this
     is super mysterious and not well documented.  sigh.  But they did mail out USB sticks to

music21/corpus/_metadataCache/core.p.gz

@@ -1396,7 +1396,8 @@ def fileFormat(self, value: str) -> None:
     @property
     def filePath(self) -> str|None:
         '''
-        Get or set the file path that was parsed.
+        Get or set the file path that was parsed.  This returns a string, not a Path object
+        that is deliberate for caching.
         '''
         return self._getSingularAttribute('filePath')
 
@@ -2481,7 +2482,7 @@ def __init__(self, **keywords):
         self.pitchHighest = None
         self.pitchLowest = None
         self.quarterLength = None
-        self.sourcePath = ''
+        self.sourcePath: str = ''
         self.tempoFirst = None
         self.tempos = []
         self.timeSignatureFirst = None

music21/metadata/__init__.py

@@ -37,6 +37,7 @@
 
 if t.TYPE_CHECKING:
     from collections.abc import Iterable
+    from music21.metadata import Metadata
 
 
 # -----------------------------------------------------------------------------
@@ -64,7 +65,8 @@ class MetadataEntry(prebase.ProtoM21Object):
     <music21.metadata.bundles.MetadataEntry 'bach_bwv66_6_mxl'>
 
     The sourcePath of the metadata entry refers to the file path at which its
-    score file is found:
+    score file is found, but it is usually a relative path to the top of the corpus
+    directory.  It is a pathlib object.
 
     >>> metadataEntry.sourcePath
     PosixPath('bach/bwv66.6.mxl')
@@ -85,21 +87,28 @@ class MetadataEntry(prebase.ProtoM21Object):
     # INITIALIZER #
 
     def __init__(self,
-                 sourcePath=None,
-                 number=None,
-                 metadataPayload=None,
-                 corpusName=None,
+                 sourcePath: str|pathlib.Path = '',
+                 number: int|None = None,
+                 metadataPayload: Metadata|None = None,
+                 corpusName: str = '',
                  ):
-        self._sourcePath = str(sourcePath)
+        # only store strings internally; sourcePath is s
+        self._sourcePath: str = str(sourcePath)
         self._number = number
         self._metadataPayload = metadataPayload
         self._corpusName = corpusName
 
     # SPECIAL METHODS #
-
     def __getnewargs__(self):
+        '''
+        This is all the information that is needed for pickling.
+        Specifically do not include _corpusName.
+
+        Note do not pickle Pathlib objects, so make sure to do ._sourcePath
+        not .sourcePath
+        '''
         return (
-            self.sourcePath,
+            self._sourcePath,
             self.metadata,
             self.number,
         )
@@ -109,7 +118,8 @@ def _reprInternal(self):
 
     def __fspath__(self):
         '''
-        for Py3.6+ to allow MetadataEntries to be used where file paths are being employed
+        Allows MetadataEntries to be used where file paths are being employed,
+        so it can be used with opening and closing, etc.
 
         Returns self.sourcePath() as a string
 
@@ -138,8 +148,14 @@ def search(self, query=None, field=None, **keywords):
 
     # PUBLIC PROPERTIES #
 
+    # we seem to be storing everything as a property so that the object
+    # is immutable after creating.
+
     @property
     def corpusPath(self):
+        '''
+        Returns the sourcePath as a string, with _number appended if it is not None.
+        '''
         return MetadataBundle.corpusPathToKey(self.sourcePath, self.number)
 
     @property
@@ -638,7 +654,7 @@ def corpus(self, newCorpus):
         self._corpus = common.wrapWeakref(newCorpus)
 
     @property
-    def filePath(self):
+    def filePath(self) -> pathlib.Path | None:
         r'''
         The filesystem name of the cached metadata bundle, if the metadata
         bundle's name is not None.
@@ -829,7 +845,7 @@ def clear(self):
         self._metadataEntries.clear()
 
     @staticmethod
-    def corpusPathToKey(filePath, number=None):
+    def corpusPathToKey(filePath: str|pathlib.Path, number: int|None = None):
         r'''
         Given a file path or corpus path, return the metadata key:
 
@@ -841,6 +857,12 @@ def corpusPathToKey(filePath, number=None):
         >>> key = mb.corpusPathToKey('corelli/opus3no1/1grave.xml')
         >>> key.endswith('corelli_opus3no1_1grave_xml')
         True
+
+        Numbers are appended if given
+
+        >>> key = mb.corpusPathToKey('corelli/opus3no1/1grave.xml', number=3)
+        >>> key.endswith('corelli_opus3no1_1grave_xml_3')
+        True
         '''
         if isinstance(filePath, pathlib.Path):
             try:
@@ -866,6 +888,7 @@ def corpusPathToKey(filePath, number=None):
             corpusPath = corpusPath.replace(os.sep, '_')
 
         corpusPath = corpusPath.replace('.', '_')
+
         # append name to metadata path
         if number is not None:
             return f'{corpusPath}_{number}'
@@ -1315,18 +1338,38 @@ def write(self, filePath=None):
         filePath = filePath or self.filePath
         if self.filePath is not None:
             filePath = self.filePath
+
             environLocal.printDebug(['MetadataBundle: writing:', filePath])
             storedCorpusClient = self._corpus  # no weakrefs allowed
             self._corpus = None
-            uncompressed = pickle.dumps(self, protocol=3)
-            # 3 is a safe protocol for some time to come.
+
+            # Protocol 5 is Python 3.8 and above.
+            uncompressed = pickle.dumps(self, protocol=5)
+
+            # # uncomment this and SafePickler when we next need to
+            # # figure out where Pathlib objects are sneaking into
+            # # the pickle -- we've had to rediagnose this too many times
+            # # to count.
+            # with open(filePath, 'wb') as f:
+            #     SafePickler(f, protocol=5).dump(self)
+            #
+            # with open(filePath, 'rb') as f:
+            #     uncompressed = f.read()  # ridiculous...
 
             with gzip.open(filePath, 'wb') as outFp:
                 outFp.write(uncompressed)
             self._corpus = storedCorpusClient
 
         return self
 
+# class SafePickler(pickle.Pickler):
+#     def persistent_id(self, obj):
+#         if isinstance(obj, pathlib.Path):
+#             raise TypeError(f'Pickling pathlib.Path objects is not allowed! {obj}')
+#         else:
+#             print(obj)
+#         return None  # Default behavior for other objects
+
 
 _test_bundles: dict[str, MetadataBundle] = {}