Merge pull request #237 from scholarly-python-package/develop

Develop

Author: programize-admin

README.md

@@ -85,6 +85,7 @@ print([citation['bib']['title'] for citation in scholarly.citedby(pub)])
 >>> author = scholarly.search_author_id('Smr99uEAAAAJ')
 >>> scholarly.pprint(author)
 {'affiliation': 'Professor of Vision Science, UC Berkeley',
+ 'email_domain': '@berkeley.edu',
  'filled': False,
  'interests': ['vision science', 'psychology', 'human factors', 'neuroscience'],
  'name': 'Martin Banks',

scholarly/author_parser.py

@@ -76,7 +76,16 @@ def _fill_basics(self, soup, author):
         author['affiliation'] = soup.find('div', class_='gsc_prf_il').text
         author['interests'] = [i.text.strip() for i in
                           soup.find_all('a', class_='gsc_prf_inta')]
-
+        if author['source'] == AuthorSource.AUTHOR_PROFILE_PAGE:
+            email = soup.find('div', id="gsc_prf_ivh", class_="gsc_prf_il")
+            if email.text != "No verified email":
+                author['email_domain'] = '@'+email.text.split(" ")[3]
+        if author['source'] == AuthorSource.CO_AUTHORS_LIST:
+            picture = soup.find('img', id="gsc_prf_pup-img").get('src')
+            if "avatar_scholar" in picture:
+                picture = _HOST.format(picture)
+            author['url_picture'] = picture
+        
     def _fill_indices(self, soup, author):
         index = soup.find_all('td', class_='gsc_rsb_std')
         if index:

scholarly/data_types.py

@@ -182,7 +182,7 @@ class Author(TypedDict, total=False):
     :param scholar_id: The id of the author on Google Scholar
     :param name: The name of the author
     :param affiliation: The affiliation of the author
-    :param email_domain: The email domain of the author (source: SEARCH_AUTHOR_SNIPPETS)
+    :param email_domain: The email domain of the author (source: SEARCH_AUTHOR_SNIPPETS, AUTHOR_PROFILE_PAGE)
     :param url_picture: The URL for the picture of the author
     :param citedby: The number of citations to all publications. (source: SEARCH_AUTHOR_SNIPPETS)
     :param filled: The set of sections filled out of the total set of sections that can be filled

scholarly/publication_parser.py

@@ -309,7 +309,12 @@ def fill(self, publication: Publication)->Publication:
             # number of citation per year
             years = [int(y.text) for y in soup.find_all(class_='gsc_vcd_g_t')]
             cites = [int(c.text) for c in soup.find_all(class_='gsc_vcd_g_al')]
-            publication['cites_per_year'] = dict(zip(years, cites))
+            cites_year = [int(c.get('href')[-4:]) for c in soup.find_all(class_='gsc_vcd_g_a')]
+            nonzero_cites_per_year = dict(zip(cites_year, cites))
+            res_dict = {}
+            for year in years:
+                res_dict[year] = (nonzero_cites_per_year[year] if year in nonzero_cites_per_year else 0)
+            publication['cites_per_year'] = res_dict
 
             if soup.find('div', class_='gsc_vcd_title_ggi'):
                 publication['eprint_url'] = soup.find(

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name='scholarly',
-    version='1.0.2',
+    version='1.0.3',
     author='Steven A. Cholewiak, Panos Ipeirotis, Victor Silva',
     author_email='steven@cholewiak.com, panos@stern.nyu.edu, vsilva@ualberta.ca',
     description='Simple access to Google Scholar authors and citations',

test_module.py

@@ -39,6 +39,7 @@ def setUp(self):
             proxy_generator.Tor_Internal(tor_cmd = tor_cmd)
             scholarly.use_proxy(proxy_generator)
         elif self.connection_method == "luminati":
+            scholarly.set_retries(10)
             proxy_generator.Luminati(usr=os.getenv("USERNAME"),passwd=os.getenv("PASSWORD"),proxy_port = os.getenv("PORT"))
             scholarly.use_proxy(proxy_generator)
         elif self.connection_method == "freeproxy":