Skip to content

Improve use of async DocParse via aryn-sdk #1134

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MarkLindblad merged 21 commits into from
Jan 29, 2025
Merged

Improve use of async DocParse via aryn-sdk #1134

Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
9d24f31
Make error code bounds checking look nicer
MarkLindblad Jan 24, 2025
d6f2ce4
Rename `_set_stream` to `_should_stream`
MarkLindblad Jan 24, 2025
82ff70d
Add comment in docstring about how cancellation can fail if the speci…
MarkLindblad Jan 24, 2025
57b2f1f
Make `partition_file_async_list` example output more concise
MarkLindblad Jan 24, 2025
5b7080a
Refactor shared header code into function, add `User-Agent`
MarkLindblad Jan 27, 2025
f14be07
Remove `test_partition_file_async`
MarkLindblad Jan 27, 2025
2cb94f8
Remove example from `partition_file_async_list` docstring
MarkLindblad Jan 27, 2025
6968d85
Keep all async examples in aryn-sdk README.md, point there from docst…
MarkLindblad Jan 27, 2025
56afd3a
Consolidate url rewriting into one function
MarkLindblad Jan 27, 2025
796c442
Add comment making behavior more obvious
MarkLindblad Jan 27, 2025
5c9e7a9
Remove dependence on `importlib`
MarkLindblad Jan 28, 2025
ec8f348
Make `_convert_sync_to_async_url` use more readable, fix linting
MarkLindblad Jan 28, 2025
ba11fb9
Simplify UX of `partition_file_async_list`
MarkLindblad Jan 28, 2025
80fca40
Filter out non-DocParse jobs from `partition_file_async_list`
MarkLindblad Jan 28, 2025
4a1d50c
Make `test_multiple_partition_file_async` more robust
MarkLindblad Jan 28, 2025
a49f7e9
Make `_convert_sync_to_async_url`'s `truncate` a keyword argument
MarkLindblad Jan 29, 2025
0f1cdb0
Improve performance of `partition_file_async_list`
MarkLindblad Jan 29, 2025
a7b7c44
Fix return description in docstring for `partition_file_async_result`
MarkLindblad Jan 29, 2025
a69af61
Improve return type in docstring of `partition_file_async_result`
MarkLindblad Jan 29, 2025
c069558
Remove repetitive sentence in docstring for `partition_file_async_res…
MarkLindblad Jan 29, 2025
7fba9f0
Fix `aryn-sdk` notebook example `ArynPartitionerPython.ipynb`
MarkLindblad Jan 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Make _convert_sync_to_async_url's truncate a keyword argument
  • Loading branch information
@MarkLindblad
MarkLindblad committed Jan 29, 2025
commit a49f7e94f66460b6bd30f66af7ff74715497cf18
14 changes: 7 additions & 7 deletions lib/aryn-sdk/aryn_sdk/partition/partition.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -378,12 +378,12 @@ def partition_file_async_submit(
if async_submit_url:
docparse_url = async_submit_url
elif not aps_url and not docparse_url:
docparse_url = _convert_sync_to_async_url(ARYN_DOCPARSE_URL, "/submit", False)
docparse_url = _convert_sync_to_async_url(ARYN_DOCPARSE_URL, "/submit", truncate=False)
else:
if aps_url:
aps_url = _convert_sync_to_async_url(aps_url, "/submit", False)
aps_url = _convert_sync_to_async_url(aps_url, "/submit", truncate=False)
if docparse_url:
docparse_url = _convert_sync_to_async_url(docparse_url, "/submit", False)
docparse_url = _convert_sync_to_async_url(docparse_url, "/submit", truncate=False)

return _partition_file_inner(
file=file,
Expand All @@ -406,7 +406,7 @@ def partition_file_async_submit(
)


def _convert_sync_to_async_url(url: str, prefix: str, truncate: bool) -> str:
def _convert_sync_to_async_url(url: str, prefix: str, *, truncate: bool) -> str:
parsed_url = urlparse(url)
assert parsed_url.path.startswith("/v1/")
if parsed_url.path.startswith("/v1/async/submit"):
Expand Down Expand Up @@ -441,7 +441,7 @@ def partition_file_async_result(
`partition_file` had the partitioning been done synchronously.
"""
if not async_result_url:
async_result_url = _convert_sync_to_async_url(ARYN_DOCPARSE_URL, "/result", True)
async_result_url = _convert_sync_to_async_url(ARYN_DOCPARSE_URL, "/result", truncate=True)

aryn_config = _process_config(aryn_api_key, aryn_config)

Expand Down Expand Up @@ -479,7 +479,7 @@ def partition_file_async_cancel(
For an example of usage see README.md
"""
if not async_cancel_url:
async_cancel_url = _convert_sync_to_async_url(ARYN_DOCPARSE_URL, "/cancel", True)
async_cancel_url = _convert_sync_to_async_url(ARYN_DOCPARSE_URL, "/cancel", truncate=True)

aryn_config = _process_config(aryn_api_key, aryn_config)

Expand Down Expand Up @@ -517,7 +517,7 @@ def partition_file_async_list(
}
"""
if not async_list_url:
async_list_url = _convert_sync_to_async_url(ARYN_DOCPARSE_URL, "/list", True)
async_list_url = _convert_sync_to_async_url(ARYN_DOCPARSE_URL, "/list", truncate=True)

aryn_config = _process_config(aryn_api_key, aryn_config)

Expand Down
Loading