Skip to content

Commit eef9a54

Browse files
sgogginssgoggins
committed
Merge remote-tracking branch 'origin/test'
2 parents 36b0dd3 + c4ef918 commit eef9a54

File tree

28 files changed

+7860
-5131
lines changed

28 files changed

+7860
-5131
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ rebuild:
5252
@ scripts/control/rebuild.sh prod
5353

5454
rebuild-dev:
55-
@ scripts/control/rebuild.sh dev
55+
@ scripts/control/rebuild-backend.sh dev
5656

5757

5858
#

‎augur/config.py‎

Lines changed: 51 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,35 @@
22
import os
33
import json
44
import logging
5+
# generate random integer values
6+
from random import seed
7+
from random import randint
8+
# seed random number generator
9+
# DO NOT create a seed.
10+
# Actual randomness is generated without a seed
11+
# for this use case.
12+
13+
contributor_breadth_worker_p = randint(48000, 48500)
14+
facade_worker_p = randint(48501, 49000)
15+
insight_worker_p = randint(49002, 49500)
16+
metric_status_worker_p = randint(49501, 50000)
17+
pull_request_worker_p = randint(50001, 50500)
18+
repo_info_worker_p = randint(50501, 51000)
19+
value_worker_p = randint(51002, 51500)
20+
contributor_worker_p = randint(52000, 52500)
21+
message_insights_worker_p = randint(53000, 53499)
22+
pull_request_analysis_worker_p = randint(54000, 54500)
23+
discourse_analysis_worker_p = randint(54500, 54999)
24+
message_insights_worker_p = randint(55000, 55499)
25+
clustering_worker_p = randint(57000, 57499)
26+
github_worker_p = randint(57500, 58000)
27+
linux_badge_worker_p = randint(47000,47499)
28+
gitlab_issues_worker_p = randint(47500,47999)
29+
release_worker_p = randint(56000, 56499)
30+
gitlab_merge_request_worker_p = randint(56500, 56999)
31+
main_port = randint(5001,5300)
32+
33+
534

635
from augur.logging import ROOT_AUGUR_DIRECTORY
736

@@ -198,23 +227,23 @@
198227
},
199228
"Workers": {
200229
"contributor_breadth_worker": {
201-
"port": 50003,
230+
"port": contributor_breadth_worker_p,
202231
"switch": 0,
203232
"workers": 1
204233
},
205234
"facade_worker": {
206-
"port": 50100,
235+
"port": facade_worker_p,
207236
"repo_directory": "repos/",
208237
"switch": 1,
209238
"workers": 1
210239
},
211240
"github_worker": {
212-
"port": 50200,
241+
"port": github_worker_p,
213242
"switch": 1,
214243
"workers": 1
215244
},
216245
"insight_worker": {
217-
"port": 50300,
246+
"port": insight_worker_p,
218247
"metrics": {"issues-new": "issues", "code-changes": "commit_count", "code-changes-lines": "added",
219248
"reviews": "pull_requests", "contributors-new": "new_contributors"},
220249
"confidence_interval": 95,
@@ -225,89 +254,89 @@
225254
"anomaly_days": 14
226255
},
227256
"linux_badge_worker": {
228-
"port": 50400,
257+
"port": linux_badge_worker_p ,
229258
"switch": 1,
230259
"workers": 1
231260
},
232261
"metric_status_worker": {
233-
"port": 50500,
262+
"port": metric_status_worker_p,
234263
"switch": 0,
235264
"workers": 1
236265
},
237266
"pull_request_worker": {
238-
"port": 50600,
267+
"port": pull_request_worker_p,
239268
"switch": 1,
240269
"workers": 1
241270
},
242271
"repo_info_worker": {
243-
"port": 50700,
272+
"port": repo_info_worker_p,
244273
"switch": 1,
245274
"workers": 1
246275
},
247276
"value_worker": {
248-
"port": 50800,
277+
"port": value_worker_p,
249278
"scc_bin": "scc",
250279
"switch": 0,
251280
"workers": 1
252281
},
253282
"contributor_worker": {
254-
"port": 50900,
283+
"port": contributor_worker_p,
255284
"switch": 0,
256285
"workers": 1
257286
},
258287
"gitlab_issues_worker": {
259-
"port": 51000,
288+
"port": gitlab_issues_worker_p,
260289
"switch": 1,
261290
"workers": 1
262291
},
263292
"release_worker": {
264-
"port": 51100,
293+
"port": release_worker_p,
265294
"switch": 1,
266295
"workers": 1
267296
},
268297
"gitlab_merge_request_worker": {
269-
"port": 51200,
298+
"port": gitlab_merge_request_worker_p,
270299
"switch": 0,
271300
"workers": 1
272301
},
273302
"message_insights_worker": {
274-
"port": 51300,
303+
"port": message_insights_worker_p,
275304
"switch": 0,
276305
"workers": 1,
277306
"insight_days": 30,
278307
"models_dir": "message_models"
279308
},
280309
"pull_request_analysis_worker": {
281-
"port": 51400,
310+
"port": pull_request_analysis_worker_p,
282311
"switch": 0,
283312
"workers": 1,
284313
"insight_days": 30
285314
},
286315
"discourse_analysis_worker":{
287-
"port" : 51500,
316+
"port" : discourse_analysis_worker_p,
288317
"switch": 0,
289318
"workers": 1
290319
},
291320
"message_insights_worker": {
292-
"port": 51300,
321+
"port": message_insights_worker_p,
293322
"switch": 0,
294323
"workers": 1,
295324
"insight_days": 30,
296325
"models_dir": "message_models"
297326
},
298327
"pull_request_analysis_worker": {
299-
"port": 51400,
328+
"port": pull_request_analysis_worker_p,
300329
"switch": 0,
301330
"workers": 1,
302331
"insight_days": 30
303332
},
304333
"discourse_analysis_worker":{
305-
"port" : 51500,
334+
"port" : discourse_analysis_worker_p,
306335
"switch": 0,
307336
"workers": 1
308337
},
309338
"clustering_worker": {
310-
"port": 51600,
339+
"port": clustering_worker_p,
311340
"switch": 0,
312341
"workers": 1,
313342
"max_df" : 0.9,
@@ -335,13 +364,13 @@
335364
"Server": {
336365
"cache_expire": "3600",
337366
"host": "0.0.0.0",
338-
"port": "5000",
367+
"port": main_port,
339368
"workers": 4,
340369
"timeout": 60
341370
},
342371
"Frontend": {
343372
"host": "0.0.0.0",
344-
"port": "5000"
373+
"port": main_port
345374
},
346375
"Logging": {
347376
"logs_directory": "logs/",

‎augur/housekeeper.py‎

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -324,17 +324,38 @@ def prep_jobs(self):
324324
def update_url_redirects(self):
325325
if 'switch' in self.update_redirects and self.update_redirects['switch'] == 1 and 'repo_group_id' in self.update_redirects:
326326
repos_urls = self.get_repos_urls(self.update_redirects['repo_group_id'])
327+
if self.update_redirects['repo_group_id'] == 0:
328+
logger.info("Repo Group Set to Zero for URL Updates")
329+
else:
330+
logger.info("Repo Group ID Specified.")
327331
for url in repos_urls:
328-
r = requests.get(url)
329-
check_for_update = url != r.url
330-
if check_for_update:
331-
self.update_repo_url(url, r.url, self.update_redirects['repo_group_id'])
332+
url = self.trim_git_suffix(url)
333+
if url:
334+
r = requests.get(url)
335+
check_for_update = url != r.url
336+
if check_for_update:
337+
self.update_repo_url(url, r.url, self.update_redirects['repo_group_id'])
338+
339+
def trim_git_suffix(self, url):
340+
if url.endswith('.git'):
341+
url = url.replace('.git', '')
342+
elif url.endswith('.github.io'):
343+
url = url.replace('.github.io', '')
344+
elif url.endswith('/.github'):
345+
url = ''
346+
return url
332347

333348
def get_repos_urls(self, repo_group_id):
334-
repos_sql = s.sql.text("""
335-
SELECT repo_git FROM repo
336-
WHERE repo_group_id = ':repo_group_id'
337-
""")
349+
if self.update_redirects['repo_group_id'] == 0:
350+
repos_sql = s.sql.text("""
351+
SELECT repo_git FROM repo
352+
""")
353+
logger.info("repo_group_id is 0")
354+
else:
355+
repos_sql = s.sql.text("""
356+
SELECT repo_git FROM repo
357+
WHERE repo_group_id = ':repo_group_id'
358+
""")
338359

339360
repos = pd.read_sql(repos_sql, self.db, params={'repo_group_id': repo_group_id})
340361

@@ -344,6 +365,13 @@ def get_repos_urls(self, repo_group_id):
344365
return repos['repo_git']
345366

346367
def update_repo_url(self, old_url, new_url, repo_group_id):
368+
trimmed_new_url = self.trim_git_suffix(new_url)
369+
if not trimmed_new_url:
370+
logger.info("New repo is named .github : {} ... skipping \n".format(new_url))
371+
return
372+
else:
373+
new_url = trimmed_new_url
374+
347375
old_repo_path = Housekeeper.parseRepoName(old_url)
348376
old_repo_group_name = old_repo_path[0]
349377
new_repo_path = Housekeeper.parseRepoName(new_url)
@@ -378,6 +406,7 @@ def update_repo_url(self, old_url, new_url, repo_group_id):
378406
new_repo_group_id = self.db.execute(insert_sql, new_repo_group_name=new_repo_group_name).fetchone()[0]
379407
logger.info("Inserted repo group {} with id {}\n".format(new_repo_group_name, new_repo_group_id))
380408

409+
new_repo_group_id = '%s' % new_repo_group_id
381410
update_sql = s.sql.text("""
382411
UPDATE repo SET repo_git = :new_url, repo_path = NULL, repo_name = NULL, repo_status = 'New', repo_group_id = :new_repo_group_id
383412
WHERE repo_git = :old_url

0 commit comments

Comments
 (0)