From 200157b01d9cc264fc1bff02757fa9ce99806706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matt=C3=A9o=20Delabre?= Date: Tue, 26 Nov 2019 10:36:07 -0500 Subject: [PATCH 1/2] =?UTF-8?q?pageviews:=20Affichage=20des=20requ=C3=AAte?= =?UTF-8?q?s=20effectu=C3=A9es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data/fetch/http.py | 35 +++++++++++++++++++++++++------ data/fetch/wikipedia_pageviews.py | 8 ++++--- data/plot_pageviews.py | 3 ++- 3 files changed, 36 insertions(+), 10 deletions(-) mode change 100644 => 100755 data/plot_pageviews.py diff --git a/data/fetch/http.py b/data/fetch/http.py index 91fdbc1..c06f5c9 100644 --- a/data/fetch/http.py +++ b/data/fetch/http.py @@ -1,15 +1,38 @@ +import logging import requests import urllib3 # Session de requêtage HTTP session = requests.Session() -# Réalise 10 essais de récupération de chaque page, en augmentant -# progressivement le délai entre chaque essai -retry = urllib3.util.Retry(total=10, backoff_factor=0.3) -adapter = requests.adapters.HTTPAdapter(max_retries=retry) -session.mount('http', adapter) -session.mount('https', adapter) +# Objet pour afficher le journal d’exécution +logger = logging.getLogger('fetch.http') + + +class LoggingAdapter(requests.adapters.HTTPAdapter): + """ + Adapteur de requêtes vérifiant le nombre d’accès réalisés par seconde. + """ + def __init__(self): + # Réessaie jusqu’à dix fois la récupération de chaque page, en + # augmentant progressivement le délai entre chaque essai + retry = urllib3.util.Retry(total=10, backoff_factor=0.3) + super(LoggingAdapter, self).__init__(max_retries=retry) + + def send( + self, request, stream=False, timeout=None, verify=True, + cert=None, proxies=None + ): + logger.debug('{} {}'.format(request.method, request.url)) + return super(LoggingAdapter, self).send( + request, stream=stream, timeout=timeout, + verify=verify, cert=cert, proxies=proxies + ) + + +adapter = LoggingAdapter() +session.mount('http://', adapter) +session.mount('https://', adapter) # Identification du robot session.headers['User-Agent'] = ( diff --git a/data/fetch/wikipedia_pageviews.py b/data/fetch/wikipedia_pageviews.py index db07a9c..66d0f1f 100644 --- a/data/fetch/wikipedia_pageviews.py +++ b/data/fetch/wikipedia_pageviews.py @@ -28,12 +28,14 @@ def _year_date_distance(a, b): """ Calcule la distance entre deux jours de l’année. - :example: _year_date_distance(10, 360) == 15 - :example: _year_date_distance(numpy.array([10, 182, 355]), 182) - == [172, 0, 173] :param a: Première valeur (peut être un tableau numpy). :param b: Seconde valeur (peut être un tableau numpy). :return: Valeur de la distance. + + >>> _year_date_distance(10, 360) + 15 + >>> _year_date_distance(numpy.array([10, 182, 355]), 182) + [172, 0, 173] """ return numpy.stack(( numpy.mod(a - b, len(year_all_days)), diff --git a/data/plot_pageviews.py b/data/plot_pageviews.py old mode 100644 new mode 100755 index e63055e..6ff2c59 --- a/data/plot_pageviews.py +++ b/data/plot_pageviews.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import calendar import collections from datetime import datetime @@ -8,7 +9,7 @@ from matplotlib import pyplot import sys # Objet pour afficher le journal d’exécution -logger = logging.getLogger('pageviews') +logger = logging.getLogger('plot_pageviews') logging.basicConfig(level=logging.INFO) if len(sys.argv) < 4: From bc271b9b6c0cdf5029cb5b72815e79228d1b3ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matt=C3=A9o=20Delabre?= Date: Tue, 26 Nov 2019 10:37:19 -0500 Subject: [PATCH 2/2] data: Correction .gitignore pour Python --- .gitignore | 12 ++++++++---- data/fetch/__pycache__/__init__.cpython-38.pyc | Bin 161 -> 0 bytes data/fetch/__pycache__/http.cpython-38.pyc | Bin 537 -> 0 bytes .../__pycache__/mediawiki_api.cpython-38.pyc | Bin 1843 -> 0 bytes .../wikipedia_pageviews.cpython-38.pyc | Bin 4143 -> 0 bytes 5 files changed, 8 insertions(+), 4 deletions(-) delete mode 100644 data/fetch/__pycache__/__init__.cpython-38.pyc delete mode 100644 data/fetch/__pycache__/http.cpython-38.pyc delete mode 100644 data/fetch/__pycache__/mediawiki_api.cpython-38.pyc delete mode 100644 data/fetch/__pycache__/wikipedia_pageviews.cpython-38.pyc diff --git a/.gitignore b/.gitignore index bc9c5f8..a996968 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ -node_modules -__pycache__ -.cache -dist +# Node.js +node_modules/ +.cache/ +dist/ + +# Python +__pycache__/ +*.py[cod] diff --git a/data/fetch/__pycache__/__init__.cpython-38.pyc b/data/fetch/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index cd19e33204b9362ba7959dd4d46d2437298d2963..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 161 zcmWIL<>g`kf`CnTVnOs{5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!HUenx(7s(x-_ zNl9wHen3%vR%&udv3_}Gc4lsBN@jARZc1ixYGQG!E|4rr&d^UuEJ@T)ODzF1~rstxbJJHVw< zU!kwjThv=myaE^MO(MjlBl(;8tog>@_-t#d!+iYs_2=uWfU&;~c^Q!>hyJ(=4~i*X zGFWF09NArP&mcrzL`AFu6{_eqfEKbMDN<~y_?AN(!@G`ZVf#MdY*i=gTBR5kUDe+A zQ->_;Jop#br-#PRWN#I}53&GaG9*jb9?AU5jK`N_nUk>@hi)amUs@!q(JiAX(H!Ci z-Oc$IeWfc@I+q`{L+Nmc(&kg~X7G0R^{VHc5ED(M91U_)iAoZoQRulv@$FgNS8I9S z5Api!rgP@d_KzkwqpSz1#p@a!IY??-mU^@Y$q=2>X0}Y+rig0NCB&b-(FMef-vy*n z$vL5Q&^doT9#Wo$X|%()d5`y^;OQTo<)3{3 diff --git a/data/fetch/__pycache__/mediawiki_api.cpython-38.pyc b/data/fetch/__pycache__/mediawiki_api.cpython-38.pyc deleted file mode 100644 index 07594c4d627961177d8274e1b9a380413dfd16b8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1843 zcmcgty^kC<6t_LIvvZpqBtXG6$chBgN@O<;&FOR^d=wE7B2EZtMRf6vojq^nE4Js7 zkUc3Pv?(Z|y6bYHKte(kblLx6L=~!ZG&D#d{5-R}x(g^Jmi)#a&(F{A_j_+XUtR4H zw4Z@a@Q8 zi}ZU>D30}M-L=YEUF1(NC6^Pwx@dkJqQ8%BUxILCLZ`$rNgpuSVAW4p>~XY9xpg8> zwBl9Hi=Bybyi_Jr4y%sK$M?PyNisW<%JA6%PcgrErTfMX3kno%)oXM@l%=fpzO|Bwe!tnKWZRM+?xsLKFPAF_@AG znbI|~&!%ib?Kuy_K`;d%lkkw;r(e@4JLnveNz_2|2aYuH`EqcjcvPHV5wiltO!(3#l9~h)1 zsk?I*MyrL*~w@*k!`ZAX7D-a~^v4}DnQbxjO>{-|`e)$9) zd>$_8a19er2T3KHQW-o5_L&3x67-Pa8aZHxkQ(tHfmU+CzyxtrHo3e_E?hPhL(EWoyXhH7DovW}c~$Ov-^(KkLl>u#n| zJM*#emHI^K^qK4-+d{XmL5u;|)zkYFaFI3gExQ>a#I)`-HXEAY9yYj8hp0T+^W^Dm zqllhT5h+v)RuElIt+{v3|mHy}uShQ7|?H^%WQD9!wdsFP)-Ro=aS-?7SN{PN76wxQ diff --git a/data/fetch/__pycache__/wikipedia_pageviews.cpython-38.pyc b/data/fetch/__pycache__/wikipedia_pageviews.cpython-38.pyc deleted file mode 100644 index cbd47279c2f008e09dd976228b12b21615bc43fc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4143 zcmb7H&2Jn@74Pb=`LZ3y`E>S!Mj|K!iJk0{-C$*-McLdSED9{5m&mlI%63osV^US^ zwZ}8kY8_xX?13X3kVoQx`4hPMg7_0!NJwzm!4VS9KXyL@jSz?@K^Ah=dbeD@I1@eBdh)T5%jYjwpm%{krp~m z#SudfI%-NN73Ue6p3;$4{jY9n-BT;e%OAy0Fp$U5QZ6@^qh7o$h0?2Aw?1dfVJ#I} z#vN5QdeMg1ip58&-|7i@Ba%AqB%u zZpbJrl1RpShlW56i6P#u$oR1DU;W9|^eT`4RK&VQG`^@^N4bxp-a@6V zeTHHWtO2`hX-B(zY-F1rHn3jwtlJ~j^*H19XLL+X(Szt_fa`vgbc#d-NfhuH3S}J; zKyfJoE{g4-n-@}{86jReA_^H?>xn`KhkvA<67wE zOm`GM#97-eSyZyM14}Nz?4SQ&c{5MNavEtZ^5yp{{HbFgccL4xgou>5fm(Jpmtmb~ zd0l8U3GF>X`|2^EmwG$pxtEyn6~fsJDy!wP1-9_qckD0ZdAv`-(m{uZLQi!EmA1O9 zYZDwf+p`A*JNs9b_V)c>TI-Cs;#pn0?^ysV8`=YVzc#Q&1Wd2^a01Nl#!5|@>ewDC zfOWLUqD;e-$5w*v4P2+u3NRbv7CKI1RZR)9dzYK%d$k)c46*C9ks zE+QAAopUJaRa6_+zjtorgqF>_P)2w1w0G|vTEs==+xCC@S5%tkdUNS+l0z+Z4~@ydc3l#* zLO0808agD9@F(Y}qP1lU6&B5P+bKPBnDPR;!Y0P9!b4=dUFon$qm3edwcd4fsCiV@ zjL+)Ke(rm7w*9>B$=A>~1#ur86a??1sNbV9%nOKjUE*G1R_+kMb#26SeZMiXsy25& zt6c)P4a}P(Yv>H@Zi{U6RH{@et%5^>Y8wy z<>^|5@U2+ITBtyQAjy-65P}~QKO7!_An;zf)HGnn;f_caaZjI|(23WQ!-FZ{j>F$d z@SQ4N2Gy8=N0}mRNQ3c&33l%TA(}`#QY!BN?>gvgMo$Y-`8OOy<=L0FQAz1H!JId$ zI9$4g5;F976em0FT3H*bUe?B1m3}oa^h~%8TPGs4A!eyd2hsWH&p5}@=R!kL-@YF7 z8xN!;6?cbUaw3AYqVjcAaG}?SjGSlDB9#2xUL?FYi`KgRfyJz+wn<#vvO&oT0&v=U zctkN?k_#H(wrzO{?b4SBOOjV$rZ7-}xp5bnhS=`OdmMensa$fZiyK0p@Tke4(!+tL zs(nt9o~46P<+A1 zs;u453q%81x;*aawlBXzCli>xA&o;fLZ7@!??~kgW1l)f`AvL$fMN`0HJZ#mzUyq} z-_2&-X@Vc>>8zXMmv){8GzdCd@L$>CO z*gkjwtl)xV*}%PPJzW@B`|b$d`}p0T+Cy(>VWctR8CrF)hpp}m z{D};buxRZ!D2M`4cLr=M>1&YW4r_y&{GDzNYT%X@)vs8Y4eqJd{RC-w{h6g_^lUY6 zk5z2KTc1Jay^2R3Ov3VUz6jdbDo~~Qj>s}JCS)-k>nNPzi;krUpGdt#rbLSD3*!tk z87U%@VgtsO6sm%2NNzN8QA1a6d^=b_JlK-4kUr6k`3 zPuT!Uj>r*?DBhU#a0*iNiJbHiilV4zBJ#|HPO3;OFXF=#Pq^gZr!js7qkchhNT6@o zR{wG;qU;1g9EaQ~9VkN}J&X>7@d0RT5wC6gq0zWnnw4xEWnU0e{caMaYdpI5DW?4k z#p+)?y=FB435oMW?6l`f4}VA-qGE^Ey3CP3@i>bTc@xv+H&7XB0S}}?-lA78MKZW4 zFH_r(dOfgp>8K)=6s}8Oz?% s