^cdpZddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z ddl mZmZmZmZmZmZmZmZmZddlmZmZddlmZejd Z Gd d ejZ!Gd d e!Z"Gddej#Z$GddZ%dZ&Gddej'j(Z)GddZ*GddZ+GddZ,GddZ-e*e*e+e,e-dZ.dZ/dZ0d%d Z1Gd!d"ej2Z3Gd#d$ej4Z5dS)&N)_)getattr)hex) encodingerrorhttpconnectionpathutilpycompaturlutilvfsworker) stringutilurlutil)lfutils\A[a-f0-9]{64}\Zc&eZdZfdZddZxZS)lfsvfsct|stjd|zt t ||dd|ddS)z9split the path at first two characters, like: XX/XXXXX...sunexpected lfs path: %srrN)_lfsrematchrProgrammingErrorsuperrjoin)selfpath __class__s 5/usr/lib/python3/dist-packages/hgext/lfs/blobstore.pyrz lfsvfs.join-s_||D!! L()Cd)JKK KVT""''QqS 48<<<Nc#Ktj|j}t t j|}g}tj||j|pd|D]D\}}|dt dkr!| fd|DEdg|fVdS)zYield (dirpath, [], oids) tuple for blobs under path Oids only exist in the root of this vfs, so dirpath is always ''. r)onerrorNrcTg|]$}t|z|z%S)rr).0fdirpaths r zlfsvfs.walk..Fs2MMMQ6<<! 3L3LMWq[MMMr) osrnormpathbaselenr normasprefixwalkreljoinextend) rrr!root prefixlenoidsdirsfilesr&s @rr-z lfsvfs.walk3s w **-d3344 $&G LLDKC 0 0'% % %   GT5ijj)G7||q   MMMM%MMMBorNN)__name__ __module__ __qualname__rr- __classcell__rs@rrr,sL===== rrc.eZdZdZdZdZddZdZdS)nullvfscdSNr#rs r__init__znullvfs.__init__M rcdS)NFr#roids rexistsznullvfs.existsPsurc`ttjtjd|z)Ns%s: No such file or directory)IOErrorerrnoENOENTr sysstrrCs rreadz nullvfs.readSs.  L Or#)rrDdatas rwritez nullvfs.write`rArr5)r6r7r8r@rErKr-rOr#rrr<r<Lsd           rr<c(eZdZdZfdZdZxZS) lfsuploadfilez+a file-like object that supports keepalive.c~tt|||d|jj|_dSNsrb)rrQr@_datarK)ruifilenamers rr@zlfsuploadfile.__init__gs4 mT""++B%@@@JO rcdSr>r#r?s r _makeprogresszlfsuploadfile._makeprogresskstr)r6r7r8__doc__r@rXr9r:s@rrQrQdsM55$$$$$rrQcVeZdZdZdZdZdZdZdZdZ dZ dd Z d Z d Z d ZdS)localzLocal blobstore for large file contents. This blobstore is used both as a cache and as a staging area for large blobs to be uploaded to the remote blobstore. c4|jd}t||_|jddrt |_n.tj |jd}t||_|j|_dS)Ns lfs/objects experimentalslfs.disableusercachelfs) svfsrrr rU configboolr<cachevfsr _usercachedir)rrepofullpath usercaches rr@zlocal.__init__vs|9>>.11(## 7  o/F G G .#IIDMM,TWf==I"9--DM'rcHt||dS)zgOpen a read-only file descriptor to the named blob, in either the usercache or the local store.rb)openrrCs rrhz local.opensDIIcNND)))rc|j|r|j|S|j|S)a{Build the path for the given blob ``oid``. If the blob exists locally, the path may point to either the usercache or the local store. If it doesn't, it will point to the local store. This is meant for situations where existing code that isn't LFS aware needs to open a blob. Generally, prefer the ``open`` method on this class. )rarErr rCs rrz local.pathsF =   $ $ +=%%c** *x}}S!!!rc|tj}d}||dd5}tj|dD]>}|||||t|z }?|Ct||kr0d}tt||t|fzt| } | |krttd |z dddn #1swxYwY||dS) znRead the blob from the remote source in chunks, verify the content, and write to this local blobstore.rwbT atomictempsizeNsXResponse length (%d) does not match Content-Length header (%d): likely server-side crashscorrupt remote lfs object: %s)hashlibsha256r r filechunkiterrOupdater+intLfsRemoteErrorrrdigestLfsCorruptionError_linktousercache) rrDsrccontent_lengthrrrpfpchunkmsgrealoids rdownloadzlocal.downloads!! XXc5TX 2 2 b+Cg>>> # # e$$$E ")c..A.AT.I.I=%QsVVtS5H5H.I%IJJJ&--//**G#~~(677#='               0 c"""""sC!DD #D c||dd5}||dddn #1swxYwY||dS)zWrite blob to local blobstore. This should only be called from the filelog during a commit or similar. As such, there is no need to verify the data. Imports from a remote store must use ``download()`` instead.rkTrlN)r rOry)rrDrNr|s rrOz local.writes XXc5TX 2 2 b HHTNNN                c"""""s ;??cTt|jts|j|ss|jtd|ztj |j ||j |dSdSdS)aLink blobs found in the user cache into this store. The server module needs to do this when it lets the client know not to upload the blob, to ensure it is always available in this store. Normally this is done implicitly when the client reads or writes the blob, but that doesn't happen when the server tells the client that it already has the blob. lfs: found %s in the usercache N) isinstancerar<r rErUnoterrlinkrrCs rlinkfromusercachezlocal.linkfromusercaches$-11 E$(//#:N:N E GLL>??#E F F F K **3//s1C1C D D D D D E E E ErcT|j|st|jtss|jt d|ztj|j ||j |dSdSdS)Ns lfs: adding %s to the usercache ) rarErr<rUrrrrr rrCs rryzlocal._linktousercaches}##C(( E M72 2  E GLL?@@3F G G G K c**DM,>,>s,C,C D D D D D  E E E ErTcP|j|s||j||}|s7t t j||kro|j td|ztj |j ||j |nF|j td|z||j||}|S)zRead blob from local blobstore.rs%lfs: found %s in the local lfs store )r rE_readrarrqrrrwrUrrrrr)rrDverifyblobs rrKz local.readsxs## 5::dmS&99D IW^D1188::;;sBB QBCCcIJJJ DM..s33TX]]35G5GHHH GLLDEEK L L L::dhV44D rcT||}|rt|||S)z0Read blob (after verifying) from the given store)rK_verify)rr rDrrs rrz local._reads.xx}}   C    rc*tj}||5}tj|dD]}|| dddn #1swxYwY|t |kS)zQIndicate whether or not the hash of the underlying file matches its name.rnroN)rqrrrhr rsrtrrw)rrDrrr|r}s rrz local.verifys!! YYs^^ %r+BW=== % % e$$$$ % % % % % % % % % % % % % % % %c&--//****s/A%%A),A)cj|j|p|j|S)zYReturns True if the local blobstore contains the requested blob, False otherwise.)rarEr rCs rhasz local.hass-}##C((@DHOOC,@,@@rN)T)r6r7r8rYr@rhrrrOrryrKrrrr#rrr[r[os   *** """(###@ # # # E E EEEE$ + + +AAAAArr[c|}t|jtr|j}tj|dr] |jjd}n#t tf$r |j}YnwxYwt|trtj |}|St|ddrtj |j Stj|S)zbCreate a friendly message for the given URLError to be used in an LfsRemoteError message. sreasonstrerrorN)rreason Exceptionr safehasattrargsAttributeError IndexErrorstrr unitolocalr strtolocalrr forcebytestr)urlerrorinstrs r_urlerrorreasonrs D(/9-- i(( 1 ![%a(FF + ! ! ![FFF ! fc " " 1(00F z4 ( (1"4=111&x000sA A('A(ceZdZdZdZdS)lfsauthhandlericF|dd}|r|d}|dkrVtd}tj||tj |||dS)zEnforces that any authentication performed is HTTP Basic Authentication. No authentication is also acceptable. zwww-authenticateNrbasics,the server must support Basic Authentication) getsplitlowerrr urlerr httperror get_full_urlr strfromlocal)rreqr|coder~headersauthreqschemes rhttp_error_401zlfsauthhandler.http_error_401!s++0$77  ]]__Q'F||~~((GHHk++$$&&)#.. trN)r6r7r8 handler_orderrr#rrrrs(MrrcDeZdZdZdZdZdZdZdZdZ dZ d Z d S) _gitlfsremotec|j}||_|\}}|d|_|jdd}|sdt jz}tj||||_ |j t| dd|_ dS)N/r]slfs.user-agentsgit-lfs/2.3.4 (Mercurial %s)r^sretry)rUauthinforstripbaseurlconfigr versionurlmodopener urlopener add_handlerr configintretry)rrcr rUrr useragents rr@z_gitlfsremote.__init__6s WLLNN~~d++ GNN?4EFF  I7$,..HIr8Y?? "">#3#3444\\&(33 rcN|t||ddS)z,Batch upload from local to remote blobstore.uploadN_batch _deduplicaterpointers fromstores r writebatchz_gitlfsremote.writebatchBs& L**IyAAAAArcN|t||ddS)z-Batch download from remote to local blostore.downloadNrrrtostores r readbatchz_gitlfsremote.readbatchFs& L**G[AAAAArc :d|D}tjtj|tj|d}d|jz}t jtj||}| dd| dd tj |j |5}|}dddn #1swxYwYn #t jj$r} t#d |j|fzt#d |jzd } | | jt#d ||fz} t)t#d t+j| z| d} ~ wt jj$rJ} t#d |jz} t)t#dt1| z| d} ~ wwxYw tj|} n@#t4$r3t)t#d|dzwxYw|jjr |jd|jztj |!"} |jdd#tI| %zd| vrtI| dd| d<|jdtjtj| dddzd}tj&|| S)zGet metadata about objects pointed by pointers for given action Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md cg|]=}tj||d>S))rDrp)r strurlrDrpr$ps rr'z/_gitlfsremote._batchrequest..PsJ   OAEEGG,,affhh ? ?   r)objects operations%s/objects/batch)rNAcceptzapplication/vnd.git-lfs+json Content-TypeNs=check that lfs serving is enabled on %s and "%s" is supporteds/the "lfs.url" config may be used to override %s)isapi=%s, action=%ssLFS HTTP error: %shint LFS error: %ss#LFS server returns invalid JSON: %sutf-8 Status: %d %s  rc|dS)NrDr#)rs rz-_gitlfsremote._batchrequest..s qxrkeyr)z: T)indent separators sort_keyscZt|tr|dS|S)Nr)rrencode)xs r encodestrz._gitlfsremote._batchrequest..encodestrs+!S!! )xx(((Hr)'r bytesurljsondumpsrrr urlreqrequest add_header contextlibclosingrrhrKrrrrrrvrrrr json_loads ValueErrorrrU debugflagdebugstatusbytestrinfostriprsorted splitlinesrapply)rractionr requestdatar batchreqrsprawjsonexhintsrresponserrs r _batchrequestz_gitlfsremote._batchrequestJs       ' J&!)!8!8    "DL0;&&xs';';+&NNH&DEEEN,JKKK #DN$7$7$A$ABB %c((** % % % % % % % % % % % % % % %{$   !<( ) IJJ, E99RWa(<&=&=f &MNND '((:+B2+F+FF {#   DEE, !"##ob&9&99    *733HH    899..))*   7   GMM/CJ6 7 7 7&sxxzz2288::G GMM'EJJvg6H6H6J6J/K/K$L$LL M M MH$$&,Y'-?-?'''# GMM#J  #-"&        y(333sV2,D C?3 D ?DD DD H BF11HAH  HH++=I(c|D]}d|vr"|dkr||dgvrd}n*(|ddd}d|D}||dd }|rRt|d d }d d ddd} | |d|z} ttd|| fzttd|dzd S)zUScans errors from objects Raises LfsRemoteError if any objects have an errorserrorractionsrscodec8i|]}||Sr#)rDrs r z6_gitlfsremote._checkforservererror..s"333Qaeeggq333roidNrVsunknownsThe object does not exists#The object was removed by the ownersValidation errorsInternal server error)riir sstatus code %dsLFS server error for "%s": %ss1LFS server error. Unsolicited response for oid %s)rrrvr) rr responsesrr rptrmaprrVerrorsr~s r_checkforservererrorz"_gitlfsremote._checkforservererrorsQ"# # H x''[((V8<<<<..DD ||H--11'3??33(333F 8F+T22A "1j*==5?,1  jj'84'?@@$6778S/I%JKKv&'A# # rc~|dg}|||fd|D}|S)zextract objects from response of the batch API response: parsed JSON object returned by batch API return response['objects'] filtered by action raise if any object has an error sobjectscDg|]}|dgv|S)r r)r$ors rr'z1_gitlfsremote._extractobjects..s8   &AEE*b,A,A"A"AA"A"A"Ar)rr)rr rrrfilteredobjectss ` r_extractobjectsz_gitlfsremote._extractobjectss`,,z2.. !!(GV<<<       rc  |d}|d|d}|d|di}tjt j|}|dkrH||s3tj td|ztd|D]?\}} | t j|t j| @ |dkrkt|j |||_d |_| d d | d |jjt%j|j|5} | d } |j } |j jr| d| jzt j| }| ddt;|z|dkr||| | nbg} | d}|sn| !|.d| }|r| d||fzdddn #1swxYwYn#tj"j#$rv}|j jr1|j d|| fztItdtKj&|||fzd}~wtj"j'$rt}tdt j(tj)*|z}tItdtW|z|d}~wwxYw|jr|j,dSdS#|jr|j,wwxYw)aCDownload or upload a single object using basic transfer protocol obj: dict, an object description returned by batch API action: string, one of ['upload', 'download'] localstore: blobstore.local See https://github.com/git-lfs/git-lfs/blob/master/docs/api/ basic-transfers.md rr shrefsheaderrdetected corrupt lfs object: %s run hg verifyrcdS)NPUTr#r#rrrz._gitlfsremote._basictransfer..sUrrzapplication/octet-streamzContent-Lengthscontent-lengthrrrrTrnrslfs %s response: %sNs%s: %s s&LFS HTTP error: %s (oid=%s, action=%s)sattempted connection to %sr)-ritemsr rrr rrrAbortrrrQrUrrN get_methodlengthrrrrhrrrrrrrrrrrKappendrrrvrrrr urllibcompat getfullurlrclose)robjr localstorerDhrefrrkvres contentlengthrUblocksrNr rrs r_basictransferz_gitlfsremote._basictransfers&k:v&**733j/&)--i<<BBDD+%%hod&;&;<< Y  $$S)) k899C?+,,  G GDAq   xq118?13E3E F F F F2 %"",TWjooc6J6JKK %2]""">3MNNN""#3W\5HIII#DN$7$7$@$@AA NS # /@ A A W7$QHH_sz9:::'.sxxzz::@@BBGHHWuzz&9K9K9M9M2N2N'O'OOPPP[((''S-@@@@F,"xx00#"! d+++ , #xx//HN!768:L!LMMM1 N N N N N N N N N N N N N N N2{$   w   3 "22!;<<*2..V<= {#   233h6G!,,W5577D!"##ob&9&99   | % ""$$$$$ % %w| % ""$$$$ %s^ BL0=EL# L0#L''L0*L'+L0.Q'0QA1N55Q A/P;;QQ''#R c dvrtjdz|}||}t d|D}i}|D].}|dd||d</t dt dd} t|d krKj t d t|tj |fzfd } j d d r5fd} tj jd| dt|d| } n| t|d} j| t d|5} | dd}d}| D]T\}}|||z }|d z }| |j t d|zU dddn #1swxYwY|dkrdkr@jt d|tj |fzdSdkrBjt d|tj |fzdSdSdS)N)rrsinvalid Git-LFS action: %sc3BK|]}|ddVdS)sizerNr)r$rs r z'_gitlfsremote._batch../s077!AEE'1%%777777rr3rrs lfs uploadingslfs downloadingrs&lfs: need to transfer %d objects (%s) c3|K|D]4}|dd}jjrodkrtd}ndkrtd}j||dt j|fzj} |d |dfVnf#tj $rT}|dkrHjtd tj ||fz|d z}Yd}~d}~wwxYw6dS) Nr3rrslfs: downloading %s (%s) rslfs: uploading %s (%s) rTrs%lfs: failed: %r (remaining retry %d) ) rrUverboserrr bytecountrr0socketrrr) r}r(objsizer~rrrr)rs rtransferz&_gitlfsremote._batch..transfer=sz  '''1--7?,, =>>9,, ;<<GLLswwvw0G0GHH   ++CDDD0000!< 199 GLL !"K L L#-#:2#>#>"F!G"QJE$HHHH  s#0CD7#A D21D22D7r]slfs.worker-enablecZjjD]}t|dddS)N close_allcdSr>r#r#rrrz7_gitlfsremote._batch..prefork..gsDr)rhandlersr)hrs rpreforkz%_gitlfsremote._batch..preforkes@0<.nsaeeFmmrr)r@c,|dSrBrrCs rrz&_gitlfsremote._batch..rs!%%--rsbytes)unittotalslfs: processed: %s rslfs: uploaded %d files (%s) rslfs: downloaded %d files (%s) )rrr rsumrrr+rUrr r7r`rr makeprogressrtr)rrr)rr rrFsizesr(topicr:r@r2progress processedblobs_onerDs` `` rrz_gitlfsremote._batch)s 1 1 1()F)OPP P%%h77&&x6BB77w77777 9 9C%(WWWa%8%8E#''&// " ")**-..    w<>>>  ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? 199""677dnY7789;&&899dnY7789 9 '&s7A1H55H9<H9ct|dd}|r8|jD]2}|t|dd1dSdS)Nrr<cdSr>r#r#rrrz'_gitlfsremote.__del__..sr)rr>r')rrr?s r__del__z_gitlfsremote.__del__smD+t44  8' 8 8 5; 557777 8 8 8 8rN) r6r7r8r@rrr rrr0rrQr#rrrr5s 4 4 4BBBBBBS4S4S4j'''R&L%L%L%\aaaF88888rrc$eZdZdZdZdZdZdS) _dummyremotez,Dummy store storing blobs to temp directory.cn|jd|j}t||_dS)Nr^)r rrr)rrcr rds rr@z_dummyremote.__init__s,8==22(##rc*t|D]}||d}||dd5}||dddn #1swxYwYdS)NT)rrkrl)rrKrDr rO)rrrrcontentr|s rrz_dummyremote.writebatchsh'' " "AnnQUUWWTn::G!%%''5T:: "b!!! " " " " " " " " " " " " " " " " "s%BB B ct|D]k}||d5}|||ddddn #1swxYwYldSrS)rr rDr)rrrrr|s rrz_dummyremote.readbatchsh'' 4 4A!%%''5)) 4R  "d333 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4s*A00A4 7A4 Nr6r7r8rYr@rrr#rrrSrSsG66$$$""" 44444rrSc$eZdZdZdZdZdZdS) _nullremotez&Null store storing blobs to /dev/null.cdSr>r#rrcr s rr@z_nullremote.__init__rArcdSr>r#rs rrz_nullremote.writebatchrArcdSr>r#rs rrz_nullremote.readbatchrArNrXr#rrrZrZsG00           rrZc.eZdZdZdZddZddZdZdS) _promptremotez)Prompt user to set lfs.url when accessed.cdSr>r#r\s rr@z_promptremote.__init__rArNc.|dSr>_prompt)rrrrUs rrz_promptremote.writebatch rc.|dSr>rc)rrrrUs rrz_promptremote.readbatchrercDtjtd)Nslfs.url needs to be configured)rr!rr?s rrdz_promptremote._promptsk!=>>???rr>)r6r7r8rYr@rrrdr#rrr`r`se33   @@@@@rr`)httpshttpsfilesnullNctj}|D]}|||<|S)z0Remove any duplicate oids that exist in the list)r sortdictrDvalues)rreducedrs rrrsAmooG  >>  rcttj|}||kr.t t d|zt ddS)Nrrr)rrqrrrwrxr)rDrVrs rrrsi'.))002233G#~~ 0 1 1C 7#$$    ~rc|jdd}tj|pd}||r|}n:t j|dr|j}n|jddpd}tj|}|jdvr|jr#|jdd d kr|xjd z c_|jpdd z|_tjt|}|j td |z|j}|tvr$tjtd |zt|||S)aremotestore factory. return a store in _storemap depending on config If ``lfs.url`` is specified, use that remote endpoint. Otherwise, try to infer the endpoint, based on the remote repository using the same path adjustments as git. As an extension, 'http' is supported as well so that ``hg serve`` works out of the box. https://github.com/git-lfs/git-lfs/blob/master/docs/api/server-discovery.md r^surlrNs _subtoppathspathssdefault)rirhrs .git/info/lfsslfs: assuming remote store: %s slfs: unknown url scheme: %s)rUrrr r r _subtoppathrrbytesrr _storemaprr!)rcremotelfsurlr r defaulturlrs rrtrts^W^^FF + +F +fm $ $C ~  ?DD  dN 3 3 ?#DD7>>(J77>3D[&&    3 3 3 (:?3B3#74#?#?4')5#9IIJO+eJ//00C GLL>??#E F F F ZF Yk!:;;fDEEE V T3 ' ''rceZdZdS)rvN)r6r7r8r#rrrvrv sDrrvceZdZdZdS)rxz~Raised when a corrupt blob is detected, aborting an operation It exists to allow specialized handling on the server side.N)r6r7r8rYr#rrrxrxs"CCCCrrxr>)6rrHrqrr(rer8mercurial.i18nrmercurial.pycompatrmercurial.noder mercurialrrrhttpconnectionmodr r r rr r vfsmodrmercurial.utilsrr largefilesrcompilerrr< httpsendfilerQr[rr basehandlerrrrSrZr`rsrrrt StorageErrorrvr!rxr#rrrs  &&&&&&                         ( ) )VZ@     f   0%2SASASASASASASASAl1112T[,.]8]8]8]8]8]8]8]8@ 44444444&          @ @ @ @ @ @ @ @"           ((((((((V     U'   CCCCCCCCCCr