Merge pull request #319 from scholarly-python-package/develop

Releasing 1.3.0

Author: ipeirotis

scholarly/_scholarly.py

@@ -16,6 +16,7 @@
 _KEYWORDSEARCH = '/citations?hl=en&view_op=search_authors&mauthors=label:{0}'
 _KEYWORDSEARCHBASE = '/citations?hl=en&view_op=search_authors&mauthors={}'
 _PUBSEARCH = '/scholar?hl=en&q={0}'
+_CITEDBYSEARCH = '/scholar?hl=en&cites={0}'
 
 
 class _Scholarly:
@@ -123,29 +124,18 @@ def search_pubs(self,
              'url_scholarbib': '/scholar?q=info:K8ZpoI6hZNoJ:scholar.google.com/&output=cite&scirp=0&hl=en'}
 
         """
-        url = _PUBSEARCH.format(requests.utils.quote(query))
-
-        yr_lo = '&as_ylo={0}'.format(year_low) if year_low is not None else ''
-        yr_hi = '&as_yhi={0}'.format(year_high) if year_high is not None else ''
-        citations = '&as_vis={0}'.format(1 - int(citations))
-        patents = '&as_sdt={0},33'.format(1 - int(patents))
-        sortby = ''
-        start = '&start={0}'.format(start_index) if start_index > 0 else ''
-
-        if sort_by == "date":
-            if include_last_year == "abstracts":
-                sortby = '&scisbd=1'
-            elif include_last_year == "everything":
-                sortby = '&scisbd=2'
-            else:
-                print("Invalid option for 'include_last_year', available options: 'everything', 'abstracts'")
-                return
-        elif sort_by != "relevance":
-            print("Invalid option for 'sort_by', available options: 'relevance', 'date'")
-            return
-            
-        # improve str below
-        url = url + yr_lo + yr_hi + citations + patents + sortby + start
+        url = _construct_url(_PUBSEARCH.format(requests.utils.quote(query)), patents=patents, citations=citations, year_low=year_low, year_high=year_high)
+        return self.__nav.search_publications(url)
+
+    def search_citedby(self, publication_id: int, **kwargs):
+        """Searches by Google Scholar publication id and returns a generator of Publication objects.
+
+        :param publication_id: Google Scholar publication id
+        :type publication_id: int
+
+        For the remaining parameters, see documentation of `search_pubs`.
+        """
+        url = _construct_url(_CITEDBYSEARCH.format(str(publication_id)), **kwargs)
         return self.__nav.search_publications(url)
 
     def search_single_pub(self, pub_title: str, filled: bool = False)->PublicationParser:
@@ -430,3 +420,33 @@ def search_org(self, name: str, fromauthor: bool = False) -> list:
 
         url = _AUTHSEARCH.format(requests.utils.quote(name))
         return self.__nav.search_organization(url, fromauthor)
+
+def _construct_url(baseurl: str, patents: bool = True,
+                    citations: bool = True, year_low: int = None,
+                    year_high: int = None, sort_by: str = "relevance",
+                    include_last_year: str = "abstracts",
+                    start_index: int = 0)-> str:
+    """Construct URL from requested parameters."""
+    url = baseurl
+
+    yr_lo = '&as_ylo={0}'.format(year_low) if year_low is not None else ''
+    yr_hi = '&as_yhi={0}'.format(year_high) if year_high is not None else ''
+    citations = '&as_vis={0}'.format(1 - int(citations))
+    patents = '&as_sdt={0},33'.format(1 - int(patents))
+    sortby = ''
+    start = '&start={0}'.format(start_index) if start_index > 0 else ''
+
+    if sort_by == "date":
+        if include_last_year == "abstracts":
+            sortby = '&scisbd=1'
+        elif include_last_year == "everything":
+            sortby = '&scisbd=2'
+        else:
+            print("Invalid option for 'include_last_year', available options: 'everything', 'abstracts'")
+            return
+    elif sort_by != "relevance":
+        print("Invalid option for 'sort_by', available options: 'relevance', 'date'")
+        return
+        
+    # improve str below
+    return url + yr_lo + yr_hi + citations + patents + sortby + start

scholarly/publication_parser.py

@@ -62,10 +62,11 @@ def _load_url(self, url: str):
 
     def _get_total_results(self):
         for x in self._soup.find_all('div', class_='gs_ab_mdw'):
-            # Decimal separator is set by Google independent of language setting
-            match = re.match(pattern=r'(^|\s*About)\s*([0-9,\.]+)', string=x.text)
+            # Accounting for different thousands separators: 
+            # comma, dot, space, apostrophe
+            match = re.match(pattern=r'(^|\s*About)\s*([0-9,\.\s’]+)', string=x.text)
             if match:
-                return int(re.sub(pattern=r'[,\.]',repl='', string=match.group(2)))
+                return int(re.sub(pattern=r'[,\.\s’]',repl='', string=match.group(2)))
         return None
 
     # Iterator protocol

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name='scholarly',
-    version='1.2.2',
+    version='1.3.0',
     author='Steven A. Cholewiak, Panos Ipeirotis, Victor Silva',
     author_email='steven@cholewiak.com, panos@stern.nyu.edu, vsilva@ualberta.ca',
     description='Simple access to Google Scholar authors and citations',

test_module.py

@@ -117,6 +117,19 @@ def test_search_pubs_citedby(self):
         cites = [c for c in scholarly.citedby(filled)]
         self.assertEqual(len(cites), filled['num_citations'])
 
+    def test_search_pubs_citedby_id(self):
+        """
+        Test querying for citations by paper ID.
+
+        The 'Machine-learned epidemiology' paper had 11 citations as of
+        June 1, 2020.
+        """
+        # Machine-learned epidemiology: real-time detection of foodborne illness at scale
+        publication_id = 2244396665447968936
+        
+        pubs = [p for p in scholarly.search_citedby(publication_id)]
+        self.assertGreaterEqual(len(pubs), 11)
+
     def test_search_keyword(self):
         """
         When we search for the keyword "3d_shape" the author
@@ -186,13 +199,17 @@ def test_search_pubs(self):
     def test_search_pubs_total_results(self):
         """
         As of February 4, 2021 there are 32 pubs that fit the search term:
-        ["naive physics" stability "3d shape"].
+        ["naive physics" stability "3d shape"], and 17'000 results that fit
+        the search term ["WIEN2k Blaha"].
 
         Check that the total results for that search term equals 32.
         """
         pubs = scholarly.search_pubs('"naive physics" stability "3d shape"')
         self.assertGreaterEqual(pubs.total_results, 32)
 
+        pubs = scholarly.search_pubs('WIEN2k Blaha')
+        self.assertGreaterEqual(pubs.total_results, 10000)
+
     def test_search_pubs_filling_publication_contents(self):
         '''
         This process  checks the process of filling a publication that is derived
@@ -232,7 +249,5 @@ def test_extract_author_id_list(self):
         self.assertTrue(author_id_list[3] == 'TEndP-sAAAAJ')
 
 
-
-
 if __name__ == '__main__':
     unittest.main()