a }c9@sdZddlZddlZddlZddlmZmZddlmZddl m Z ddl m Z ddl mZdZd d ZGd d d ZGd ddeZGdddeZdS)u:class:`.RateLimiter` and :class:`.AsyncRateLimiter` allow to perform bulk operations while gracefully handling error responses and adding delays when needed. In the example below a delay of 1 second (``min_delay_seconds=1``) will be added between each pair of ``geolocator.geocode`` calls; all :class:`geopy.exc.GeocoderServiceError` exceptions will be retried (up to ``max_retries`` times):: import pandas as pd df = pd.DataFrame({'name': ['paris', 'berlin', 'london']}) from geopy.geocoders import Nominatim geolocator = Nominatim(user_agent="specify_your_app_name_here") from geopy.extra.rate_limiter import RateLimiter geocode = RateLimiter(geolocator.geocode, min_delay_seconds=1) df['location'] = df['name'].apply(geocode) df['point'] = df['location'].apply(lambda loc: tuple(loc.point) if loc else None) This would produce the following DataFrame:: >>> df name location \ 0 paris (Paris, Île-de-France, France métropolitaine, ... 1 berlin (Berlin, 10117, Deutschland, (52.5170365, 13.3... 2 london (London, Greater London, England, SW1A 2DU, UK... point 0 (48.8566101, 2.3514992, 0.0) 1 (52.5170365, 13.3888599, 0.0) 2 (51.5073219, -0.1276474, 0.0) To pass extra options to the `geocode` call:: from functools import partial df['location'] = df['name'].apply(partial(geocode, language='de')) To see a progress bar:: from tqdm import tqdm tqdm.pandas() df['location'] = df['name'].progress_apply(geocode) Before using rate limiting classes, please consult with the Geocoding service ToS, which might explicitly consider bulk requests (even throttled) a violation. N)chaincount)sleep default_timer)GeocoderServiceError)logger)AsyncRateLimiter RateLimitercCstddt|DdgS)z-list(_is_last_gen(2)) -> [False, False, True]css|] }dVqdS)FN).0_r r T/var/www/html/django/DPS/env/lib/python3.9/site-packages/geopy/extra/rate_limiter.py Bz_is_last_gen..T)rrange)rr r r _is_last_gen@src@s>eZdZdZefZddZddZddZdd Z d d Z d S) BaseRateLimiterz9Base Rate Limiter class for both sync and async versions.cCs8||_||_||_||_|dks$Jt|_d|_dSNr)min_delay_seconds max_retriesswallow_exceptionsreturn_value_on_exception threadingLock_lock _last_call)selfrrrrr r r__init__Js  zBaseRateLimiter.__init__cCstSNr)rr r r_clock\szBaseRateLimiter._clockccs|jj|}|jdur0||_WddS||j}|j|}|dkrb||_WddSWdn1sv0Y|VqdSr)rr rr)rclockZseconds_since_last_callwaitr r r_acquire_request_slot_gen_s   .z)BaseRateLimiter._acquire_request_slot_genc cs~ttt|jD]f\}}z |VWnL|jyp|r@dVn,tjt|jd||j||dddVYqYq0dSqdS)NTzB caught an error, retrying (%s/%s tries). Called with (*%r, **%r).exc_infoF) ziprrr_retry_exceptionsrwarningtype__name__)rargskwargsiZ is_last_tryr r r _retries_gens"    zBaseRateLimiter._retries_gencCs2|jr,tjt|jd|j||dd|jSdS)Nz> swallowed an error after %r retries. Called with (*%r, **%r).Tr$)rrr(r)r*rr)rr+r,r r r _handle_excs zBaseRateLimiter._handle_excN) r* __module__ __qualname____doc__rr'rr r#r.r/r r r rrEs&rcsFeZdZdZddddddfdd Zd d Zd d ZddZZS)r aThis is a Rate Limiter implementation for synchronous functions (like geocoders with the default :class:`geopy.adapters.BaseSyncAdapter`). Examples:: from geopy.extra.rate_limiter import RateLimiter from geopy.geocoders import Nominatim geolocator = Nominatim(user_agent="specify_your_app_name_here") search = ["moscow", "paris", "berlin", "tokyo", "beijing"] geocode = RateLimiter(geolocator.geocode, min_delay_seconds=1) locations = [geocode(s) for s in search] search = [ (55.47, 37.32), (48.85, 2.35), (52.51, 13.38), (34.69, 139.40), (39.90, 116.39) ] reverse = RateLimiter(geolocator.reverse, min_delay_seconds=1) locations = [reverse(s) for s in search] RateLimiter class is thread-safe. If geocoding service's responses are slower than `min_delay_seconds`, then you can benefit from parallelizing the work:: import concurrent.futures geolocator = OpenMapQuest(api_key="...") geocode = RateLimiter(geolocator.geocode, min_delay_seconds=1/20) with concurrent.futures.ThreadPoolExecutor() as e: locations = list(e.map(geocode, search)) .. versionchanged:: 2.0 Added thread-safety support. @TNrrerror_wait_secondsrrcs<tj||||d||_||_||ks,J|dks8JdSa  :param callable func: A function which should be wrapped by the rate limiter. :param float min_delay_seconds: Minimum delay in seconds between the wrapped ``func`` calls. To convert :abbr:`RPS (Requests Per Second)` rate to ``min_delay_seconds`` you need to divide 1 by RPS. For example, if you need to keep the rate at 20 RPS, you can use ``min_delay_seconds=1/20``. :param int max_retries: Number of retries on exceptions. Only :class:`geopy.exc.GeocoderServiceError` exceptions are retried -- others are always re-raised. ``max_retries + 1`` requests would be performed at max per query. Set ``max_retries=0`` to disable retries. :param float error_wait_seconds: Time to wait between retries after errors. Must be greater or equal to ``min_delay_seconds``. :param bool swallow_exceptions: Should an exception be swallowed after retries? If not, it will be re-raised. If yes, the ``return_value_on_exception`` will be returned. :param return_value_on_exception: Value to return on failure when ``swallow_exceptions=True``. )rrrrrNsuperrfuncr7rr;rrr7rr __class__r rrs) zRateLimiter.__init__cCs"tt|jd|t|dSNz sleep(%r))rdebugr)r*rrsecondsr r r_sleepszRateLimiter._sleepcCs|D]}||qdSrr#rCrr"r r r_acquire_request_slot s z!RateLimiter._acquire_request_slotc Os|||}|D]}|z,|j|i|}t|r@td|WS|jy}z>||r|||WYd}~S| |j WYd}~qd}~00qt ddS)NzoAn async awaitable has been passed to `RateLimiter`. Use `AsyncRateLimiter` instead, which supports awaitables.Should not have been reached) r.rFr;inspect isawaitable ValueErrorr'throwr/rCr7 RuntimeError)rr+r,genr reser r r__call__ s    $zRateLimiter.__call__ r*r0r1r2rrCrFrP __classcell__r r r=rr s)4r csFeZdZdZddddddfdd Zd d Zd d ZddZZS)r aThis is a Rate Limiter implementation for asynchronous functions (like geocoders with :class:`geopy.adapters.BaseAsyncAdapter`). Examples:: from geopy.adapters import AioHTTPAdapter from geopy.extra.rate_limiter import AsyncRateLimiter from geopy.geocoders import Nominatim async with Nominatim( user_agent="specify_your_app_name_here", adapter_factory=AioHTTPAdapter, ) as geolocator: search = ["moscow", "paris", "berlin", "tokyo", "beijing"] geocode = AsyncRateLimiter(geolocator.geocode, min_delay_seconds=1) locations = [await geocode(s) for s in search] search = [ (55.47, 37.32), (48.85, 2.35), (52.51, 13.38), (34.69, 139.40), (39.90, 116.39) ] reverse = AsyncRateLimiter(geolocator.reverse, min_delay_seconds=1) locations = [await reverse(s) for s in search] AsyncRateLimiter class is safe to use across multiple concurrent tasks. If geocoding service's responses are slower than `min_delay_seconds`, then you can benefit from parallelizing the work:: import asyncio async with OpenMapQuest( api_key="...", adapter_factory=AioHTTPAdapter ) as geolocator: geocode = AsyncRateLimiter(geolocator.geocode, min_delay_seconds=1/20) locations = await asyncio.gather(*(geocode(s) for s in search)) .. versionadded:: 2.0 r3r4r5TNr6cs<tj||||d||_||_||ks,J|dks8JdSr8r9r<r=r rrLs) zAsyncRateLimiter.__init__cs*tt|jd|t|IdHdSr?)rr@r)r*asynciorrAr r rrCszAsyncRateLimiter._sleepcs"|D]}||IdHqdSrrDrEr r rrFs z&AsyncRateLimiter._acquire_request_slotc s|||}|D]}|IdHz|j|i|IdHWS|jy}zD||rv|||WYd}~S||jIdHWYd}~qd}~00qtddS)NrG) r.rFr;r'rKr/rCr7rL)rr+r,rMr rOr r rrPs  *zAsyncRateLimiter.__call__rQr r r=rr "s-4r )r2rSrHr itertoolsrrtimerZtimeitrZ geopy.excrZ geopy.utilr__all__rrr r r r r rs2    fw