­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ 3 ybpP@s ddlZddlZddlZddlZddlZddlZddlmZmZm Z ddl mZddl m Z m Z ddl mZdd Zd d ZGd d d eZGdddeZGdddeZGdddeZGdddeZGdddeZGdddeZGdddeZGdddeZGdddeZdS) N)NSIndex ChunkIndexChunkIndexEntry) hashindex)IntegrityCheckedFileFileIntegrityError) BaseTestCasecCstd|dS)Nz%-0.32dascii)bytes)xr!/usr/lib64/python3.6/hashindex.pyHsrcCstjt|jS)N)hashlibsha256rdigest)r rrrH2src@sDeZdZddZddZddZddZd d Zd d Zd dZ dS)HashIndexTestCasec CsZ|}|jt|dx tdD]}|||t|<q W|jt|dx(tdD]}|j|t|||qRWx$tdD]}||d|t|<q|W|jt|dx,tdD] }|j|t|||dqWxtdD]}|t|=qWxtddD]}qWxtdD]}qWx&tdD]}|jt|jt|q$W|jt|dtj}|j |j ~t |j d }|jt j |jj|WdQRX|j|j }|jt|dx0tddD]"}|j|t|||dqW|j|jt|d|j |j ~|jt|j|j d|}|jtd|d|jtd|d~dS)Nrdr2rb*) assert_equallenranger assert_raisesKeyError __delitem__tempfileNamedTemporaryFilewritenameopenrrreadZ hexdigestclear setdefault)selfclsZ make_valueZshaidxr idx_namefdrrr _generic_testsN  $ " zHashIndexTestCase._generic_testcCs|jtddddS)NcSs||fS)Nr)r rrr[sz0HashIndexTestCase.test_nsindex..Z@85f72b036c692c8266e4f51ccf0cff2147204282b5e316ae508d30a448d88fef)r.r)r)rrr test_nsindexZs zHashIndexTestCase.test_nsindexcCs|jtddddS)NcSs |||fS)Nr)r rrrr/_sz3HashIndexTestCase.test_chunkindex..Z@c83fdf33755fc37879285f2ecfc5d1f63b97577494902126b6fb6f3e4d852488)r.r)r)rrrtest_chunkindex^s z!HashIndexTestCase.test_chunkindexcCsd}tj}t}|j|jtjj|j}|jt |dx t |D]}||f|t |<qFW|j|jxt |D]}|t |=qtW|jt |d|j|j|j|tjj|jdS)Nir) r!r"rr#r$ospathgetsizerrrr)r)nr,r+Z initial_sizer rrr test_resizebs   zHashIndexTestCase.test_resizecCst}x tdD]}||f|t|<qW|j}t|}|jt|d|jtt |t|j|ddd}|jt|d|j||dddS)Nr1r)Zmarkerr) rrrZ iteritemslistrrr StopIterationnext)r)r+r iteratorallZ second_halfrrrtest_iteritemsssz HashIndexTestCase.test_iteritemscCsbt}d |td<d |td<d |td<t}d|td<d|td<d|td<|j|dS)Nr rr,)r rr)rr>r>)r?r@r@)rArr)rBr>r>)rCrDrD)rrmerge)r)idx1idx2rrrtest_chunkindex_merges       z'HashIndexTestCase.test_chunkindex_mergecCsBt}d |td<d |td<d |td<|j\}}}}}}dS) Nr rrr>r? ,)r rIr)rrJr>)r?rKrL)rrZ summarize)r)r+sizeZcsizeZ unique_sizeZ unique_csizeZ unique_chunksZchunksrrrtest_chunkindex_summarizes    z+HashIndexTestCase.test_chunkindex_summarizeN) __name__ __module__ __qualname__r.r0r1r6r=rHrNrrrrrs= rc@seZdZdZddZdS)HashIndexExtraTestCasezRThese tests are separate because they should not become part of the selftest. cCstdtjd}t|}ddt|D}|dd|d|d|dd}}x"t|D]\}}|||f||<q`Wx"t|D]\}}|||f||<qWx|D] }||=qWxt|D]\}}qWx |D]}qWx|D] }||=qWdS)Ni cSsg|]}tjt|jqSr)rrrr).0krrr sz=HashIndexExtraTestCase.test_chunk_indexer..rrr?)intrZMAX_LOAD_FACTORr enumerate)r)Z key_countindexZall_keyskeysZto_delete_keysikeyrrrtest_chunk_indexers"*     z)HashIndexExtraTestCase.test_chunk_indexerN)rOrPrQ__doc__r]rrrrrRsrRc@seZdZddZddZdS)HashIndexSizeTestCasecCs t}dS)N)r)r)r+rrrtest_size_on_disksz'HashIndexSizeTestCase.test_size_on_diskc Csdt}x*tdD]}||d|df|t|<qWtj }|j|jtjj |j}WdQRXdS)Nirr?) rrrr!r"r#r$r2r3r4)r)r+r[filerMrrrtest_size_on_disk_accurates  z0HashIndexSizeTestCase.test_size_on_disk_accurateN)rOrPrQr`rbrrrrr_sr_c@steZdZddZddZddZddZd d Zd d Zd dZ ddZ ddZ ddZ ddZ ddZddZdS)HashIndexRefcountingTestCasecCsjt}tjdddf|td<x"tdD]}|jtd^}}q(Wx"tdD]}|jtd^}}qLWdS)Nr rrB)r MAX_VALUErrincrefdecref)r)r+r[refcount_rrrtest_chunkindex_limitsz2HashIndexRefcountingTestCase.test_chunkindex_limitcCsdd}|||}|S)NcSsNt}|ddf|td<t}|ddf|td<|j||td^}}|S)Nr r)rrrE)Z refcount1Z refcount2rFrGrgrhrrrrEs z2HashIndexRefcountingTestCase._merge..merger)r)Z refcountaZ refcountbrEresultrrr_merges z#HashIndexRefcountingTestCase._mergecCstjd}dS)Nr)rrd)r)halfrrrtest_chunkindex_merge_limit1s z9HashIndexRefcountingTestCase.test_chunkindex_merge_limit1cCsdS)Nr)r)rrrtest_chunkindex_merge_limit2sz9HashIndexRefcountingTestCase.test_chunkindex_merge_limit2cCstjd}dS)Nr)rrd)r)rlrrrtest_chunkindex_merge_limit3s z9HashIndexRefcountingTestCase.test_chunkindex_merge_limit3cCstjd}dS)Nr)rrd)r)rlrrrtest_chunkindex_merge_limit4s z9HashIndexRefcountingTestCase.test_chunkindex_merge_limit4cCs2t}|jtdddd|jtdddddS)Nr rBrCrr?)raddr)r)rFrrrtest_chunkindex_addsz0HashIndexRefcountingTestCase.test_chunkindex_addcCs<t}tjddf|td<|jtd|td^}}dS)NrCrqr )rrdrre)r)rFrgrhrrrtest_incref_limits z.HashIndexRefcountingTestCase.test_incref_limitcCs<t}tjddf|td<|jtd|td^}}dS)NrCrqr )rrdrrf)r)rFrgrhrrrtest_decref_limit s z.HashIndexRefcountingTestCase.test_decref_limitc Cs:t}d|td<|jt|jtdWdQRXdS)Nrr )rrr)rrrAssertionErrorrf)r)rFrrrtest_decref_zeros  z-HashIndexRefcountingTestCase.test_decref_zerocCs:t}|jtdddd|jtd|jtddS)Nr rBrCrq)rrrrrerf)r)rFrrrtest_incref_decrefs z/HashIndexRefcountingTestCase.test_incref_decrefc Cs8t}|jttjdddf|td<WdQRXdS)Nr r)rrrvrdr)r)rFrrrtest_setitem_raises#s z0HashIndexRefcountingTestCase.test_setitem_raisesc!Cst}|jt|jtdWdQRX|jt|jtdWdQRX|jt|tdWdQRX|jt|jtddddWdQRXdS)Nr r)rrrrerrf OverflowErrorrr)r)r+rrr test_keyerror(s    z*HashIndexRefcountingTestCase.test_keyerrorN)rOrPrQrirkrmrnrorprsrtrurwrxryr|rrrrrcs  rcc@s@eZdZdZddZddZddZdd Zd d Zd d Z dS)HashIndexDataTestCases(eJzt0L0NgmAUhtHLT0LDEI6AuAEhMVYmVnSuYefC7AB3Aj9KNedJbnfyFne6P67P27w0EdG1Eac+Cm1ZybAsy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7LsL9nhc+cqTZ3XlO2Ys++Du5fX+l1/YFmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVn2/+0O2rYccw==cCsPtj>}tjj|d}|j|t|d}|j|jSQRXWdQRXdS)Nr+r) r!TemporaryDirectoryr2r3joinr#r%_packr&)r)r+tempdirrafrrr_serialize_hashindex:s    z*HashIndexDataTestCase._serialize_hashindexcCsRtj@}tjj|d}t|d}|j|j|WdQRXtj |SQRXdS)Nr+wb) r!r~r2r3rr%r#_unpackrr&)r) bytestringrrarrrr_deserialize_hashindexAs   z,HashIndexDataTestCase._deserialize_hashindexcCstjtj|S)N)base64Z b64encodezlibcompress)r)rrrrrHszHashIndexDataTestCase._packcCstjtj|S)N)rZ decompressrZ b64decode)r)rrrrrKszHashIndexDataTestCase._unpackcCs8t}d|td<d |td<d |td<|j|}dS) Nr rr?rxl)r rr?l)rrr)rrr)rrr)r)rFZ serializedrrrtest_identical_creationNs     z-HashIndexDataTestCase.test_identical_creationcCs,|j|j}t}d |td<|j|dS) Nr i@rCrqr?l)rrCrq)r HASHINDEXrrrE)r)rFrGrrrtest_read_known_goodWs    z*HashIndexDataTestCase.test_read_known_goodN) rOrPrQrrrrrrrrrrrr}4s r}c@seZdZddZddZdS)HashIndexIntegrityTestCasec CsJ|j|j}tjj|d}t|dd}|j|WdQRX|j}||fS)Nr+T)r3r#)rrr2r3rrr#integrity_data)r)rr+rar-rrrrwrite_integrity_checked_indexds  z8HashIndexIntegrityTestCase.write_integrity_checked_indexcCstjn}|j|\}}t|d}|jdWdQRX|jt*t|d|d}tj |WdQRXWdQRXWdQRXdS)Nzr+bsFooF)r3r#r) r!r~rr%r#rrrrr&)r)rrarr-rrrtest_integrity_checked_filens   z6HashIndexIntegrityTestCase.test_integrity_checked_fileN)rOrPrQrrrrrrrcs rc@steZdZddZddZddZddZd d Zd d Zd dZ ddZ ddZ ddZ ddZ ddZddZdS)HashIndexCompactTestCasecCsdtj}|jd|j|jdd|j|jdd|jdjdd|jdjdd||_dS)NsBORG_IDXrAlittlerr r? )ioBytesIOr#to_bytes index_data)r) num_entries num_bucketsrrrrrYys zHashIndexCompactTestCase.indexcCs|jjdtj|j}|S)Nr)rseekrr&)r)rYrrrindex_from_datas  z(HashIndexCompactTestCase.index_from_datacCstj}|j||jS)N)rrr#getvalue)r)rYdatarrr index_to_datas z&HashIndexCompactTestCase.index_to_datacCs|j}|j|j|}|S)N)rcompactr)r)rY compact_indexrrrindex_from_data_compact_to_datas z8HashIndexCompactTestCase.index_from_data_compact_to_datacGs2|jj|x |D]}|jj|jddqWdS)NrAr)rr#r)r)r\valuesvaluerrr write_entrys  z$HashIndexCompactTestCase.write_entrycCs|j|ddddS)Nlr)r)r)r\rrr write_emptysz$HashIndexCompactTestCase.write_emptycCs|j|ddddS)Nlr)r)r)r\rrr write_deletedsz&HashIndexCompactTestCase.write_deletedcCs|jddd|jtdddd|jtd|jtd|jtdddd|jtd d d d |jtd|j}|jddd|jtdddd|jtdddd|jtd d d d dS) Nr?rC)rrrr rrBrqrA rS)rYrrrrr)r)rrrr test_simplesz$HashIndexCompactTestCase.test_simplecCs|jddd|jtd|jtdddd|jtd|jtdddd|jtd d d d |jtd|j}|jddd|jtdddd|jtdddd|jtd d d d dS) Nr?rC)rrr rrrBrqrArrrS)rYrrrrr)r)rrrrtest_first_emptysz)HashIndexCompactTestCase.test_first_emptycCs|jddd|jtd|jtdddd|jtd|jtdddd|jtd|jtd d d d |j}|jddd|jtdddd|jtdddd|jtd d d d dS) Nr?rC)rrr rrrBrqrArrrS)rYrrrrr)r)rrrrtest_last_usedsz'HashIndexCompactTestCase.test_last_usedcCs|jddd|jtd|jtdddd|jtdddd|jtd|jtd|jtd d d d |j}|jddd|jtdddd|jtdddd|jtd d d d dS) Nr?rC)rrr rrrBrqrArrrS)rYrrrrr)r)rrrrtest_too_few_empty_slotssz1HashIndexCompactTestCase.test_too_few_empty_slotscCs||jddd|jtd|jtd|jtd|jtd|jtd|jtd|j}|jddddS) NrrC)rrr r?rrBrA)rYrrrr)r)rrrr test_emptysz#HashIndexCompactTestCase.test_emptycCsFt}t}d|td<d|td<d |td<|j|j|dS) Nr rrr>r?,)r rr)rr>r>)r?rr)rrrrE)r)ZmasterrFrrr test_merges    z#HashIndexCompactTestCase.test_mergeN)rOrPrQrYrrrrrrrrrrrrrrrrrxsrc@seZdZddZdS)NSIndexTestCasec CsHt}|jttjddf|td<WdQRXtjdf|td<dS)Nr rr)rrrvrdr)r)r+rrrtest_nsindex_segment_limits   z*NSIndexTestCase.test_nsindex_segment_limitN)rOrPrQrrrrrrsrc@seZdZddZdS)AllIndexTestCasecCsdS)Nr)r)rrrtest_max_load_factorsz%AllIndexTestCase.test_max_load_factorN)rOrPrQrrrrrr src@seZdZddZdS)IndexCorruptionTestCasecsddlmfddtx"tdD]}d|fd|<q(WxtdD]}d|=qLWx"tdD]}d|fd|<qjWdS) Nr)packcsd||dddS)Nz.HHiiiJiX)structrrr)r)rr)rr+rr test_bug_4829s   z%IndexCorruptionTestCase.test_bug_4829N)rOrPrQrrrrrrsr)rrrr2r!rrrrrZcrypto.file_integrityrrr rrrrRr_rcr}rrrrrrrrrs.  f/