-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathbatch.go
More file actions
1904 lines (1740 loc) · 71.1 KB
/
Copy pathbatch.go
File metadata and controls
1904 lines (1740 loc) · 71.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
package contextdev
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"slices"
"github.com/context-dot-dev/context-go-sdk/v2/internal/apijson"
"github.com/context-dot-dev/context-go-sdk/v2/internal/apiquery"
"github.com/context-dot-dev/context-go-sdk/v2/internal/requestconfig"
"github.com/context-dot-dev/context-go-sdk/v2/option"
"github.com/context-dot-dev/context-go-sdk/v2/packages/param"
"github.com/context-dot-dev/context-go-sdk/v2/packages/respjson"
"github.com/context-dot-dev/context-go-sdk/v2/shared/constant"
)
// BatchService contains methods and other services that help with interacting with
// the context.dev API.
//
// Note, unlike clients, this service does not read variables from the environment
// automatically. You should not instantiate this service directly, and instead use
// the [NewBatchService] method instead.
type BatchService struct {
options []option.RequestOption
}
// NewBatchService generates a new service that applies the given options to each
// request. These options are applied after the parent client's options (if there
// is one), and before any request-specific options.
func NewBatchService(opts ...option.RequestOption) (r BatchService) {
r = BatchService{}
r.options = opts
return
}
// Check progress, and get download links once the batch finishes.
func (r *BatchService) Get(ctx context.Context, batchID string, opts ...option.RequestOption) (res *BatchGetResponse, err error) {
opts = slices.Concat(r.options, opts)
if batchID == "" {
err = errors.New("missing required batch_id parameter")
return nil, err
}
path := fmt.Sprintf("batch/%s", url.PathEscape(batchID))
err = requestconfig.ExecuteNewRequest(ctx, http.MethodGet, path, nil, &res, opts...)
return res, err
}
// List your batches from newest to oldest. Filter by status or continue with a
// cursor.
func (r *BatchService) List(ctx context.Context, query BatchListParams, opts ...option.RequestOption) (res *BatchListResponse, err error) {
opts = slices.Concat(r.options, opts)
path := "batch/list"
err = requestconfig.ExecuteNewRequest(ctx, http.MethodGet, path, query, &res, opts...)
return res, err
}
// Stop a batch from starting new pages. In-progress pages finish, and unused
// credits are refunded.
func (r *BatchService) Cancel(ctx context.Context, batchID string, opts ...option.RequestOption) (res *BatchCancelResponse, err error) {
opts = slices.Concat(r.options, opts)
if batchID == "" {
err = errors.New("missing required batch_id parameter")
return nil, err
}
path := fmt.Sprintf("batch/%s/cancel", url.PathEscape(batchID))
err = requestconfig.ExecuteNewRequest(ctx, http.MethodPost, path, nil, &res, opts...)
return res, err
}
// Page through a finished batch's results as JSON instead of downloading the
// NDJSON files.
func (r *BatchService) GetResults(ctx context.Context, batchID string, query BatchGetResultsParams, opts ...option.RequestOption) (res *BatchGetResultsResponse, err error) {
opts = slices.Concat(r.options, opts)
if batchID == "" {
err = errors.New("missing required batch_id parameter")
return nil, err
}
path := fmt.Sprintf("batch/%s/results", url.PathEscape(batchID))
err = requestconfig.ExecuteNewRequest(ctx, http.MethodGet, path, query, &res, opts...)
return res, err
}
// Retrieve and normalize a person profile from identifiers.
func (r *BatchService) Submit(ctx context.Context, body BatchSubmitParams, opts ...option.RequestOption) (res *BatchSubmitResponse, err error) {
opts = slices.Concat(r.options, opts)
path := "people/retrieve"
err = requestconfig.ExecuteNewRequest(ctx, http.MethodPost, path, body, &res, opts...)
return res, err
}
// Page failures sharing one error code.
type PageErrorCount struct {
// Error code for these failures.
Code string `json:"code" api:"required"`
// Pages that failed with this code.
Count int64 `json:"count" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
Code respjson.Field
Count respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r PageErrorCount) RawJSON() string { return r.JSON.raw }
func (r *PageErrorCount) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// A failure of the batch as a whole, distinct from the per-page failures in
// `page_errors`.
type Failure struct {
// Why the batch itself stopped.
Code string `json:"code" api:"required"`
// Human-readable explanation.
Message string `json:"message" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
Code respjson.Field
Message respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r Failure) RawJSON() string { return r.JSON.raw }
func (r *Failure) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// The crawl controls as submitted, so the limits requested can be compared against
// what the crawl reached.
type CrawlControls struct {
// Whether links to subdomains were followed. Always false for a sitemap crawl.
FollowSubdomains bool `json:"follow_subdomains" api:"required"`
// Link depth limit. Always 0 for a sitemap crawl, which never follows links off
// its URLs; null when a `start_url` crawl set no limit.
MaxDepth int64 `json:"max_depth" api:"required"`
// The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only the URLs
// its sitemap actually lists, up to this many, so `input.reserved` is often lower.
MaxPages int64 `json:"max_pages" api:"required"`
// Where the crawl started.
Source CrawlControlsSourceUnion `json:"source" api:"required"`
// RE2 pattern URLs had to match to be crawled. Null when the crawl set none.
URLPattern string `json:"url_pattern" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
FollowSubdomains respjson.Field
MaxDepth respjson.Field
MaxPages respjson.Field
Source respjson.Field
URLPattern respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r CrawlControls) RawJSON() string { return r.JSON.raw }
func (r *CrawlControls) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// CrawlControlsSourceUnion contains all possible properties and values from
// [CrawlControlsSourceObject], [CrawlControlsSourceObject2].
//
// Use the methods beginning with 'As' to cast the union to one of its variants.
type CrawlControlsSourceUnion struct {
Type string `json:"type"`
// This field is from variant [CrawlControlsSourceObject].
URL string `json:"url"`
// This field is from variant [CrawlControlsSourceObject2].
Domain string `json:"domain"`
JSON struct {
Type respjson.Field
URL respjson.Field
Domain respjson.Field
raw string
} `json:"-"`
}
func (u CrawlControlsSourceUnion) AsCrawlControlsSourceObject() (v CrawlControlsSourceObject) {
apijson.UnmarshalRoot(json.RawMessage(u.JSON.raw), &v)
return
}
func (u CrawlControlsSourceUnion) AsCrawlControlsSourceObject2() (v CrawlControlsSourceObject2) {
apijson.UnmarshalRoot(json.RawMessage(u.JSON.raw), &v)
return
}
// Returns the unmodified JSON received from the API
func (u CrawlControlsSourceUnion) RawJSON() string { return u.JSON.raw }
func (r *CrawlControlsSourceUnion) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
type CrawlControlsSourceObject struct {
// Any of "start_url".
Type string `json:"type" api:"required"`
// Page the crawl started from.
URL string `json:"url" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
Type respjson.Field
URL respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r CrawlControlsSourceObject) RawJSON() string { return r.JSON.raw }
func (r *CrawlControlsSourceObject) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
type CrawlControlsSourceObject2 struct {
// Domain whose sitemap supplied the pages.
Domain string `json:"domain" api:"required"`
// Any of "sitemap".
Type string `json:"type" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
Domain respjson.Field
Type respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r CrawlControlsSourceObject2) RawJSON() string { return r.JSON.raw }
func (r *CrawlControlsSourceObject2) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// What submission took in, and what it charged for.
type Intake struct {
// URLs dropped before reserving because another entry resolved to the same page.
// Non-zero for sitemap crawls too, whose sitemaps routinely list a page more than
// once.
Duplicates int64 `json:"duplicates" api:"required"`
// URLs from your list rejected as unusable; the same ones are itemised in
// `invalid_urls` at submission. Null for a crawl — a crawl that resolves no usable
// page is rejected outright with a 400 rather than accepted with an empty list.
Invalid int64 `json:"invalid" api:"required"`
// Pages credits were reserved for. Everything else — progress, the refund, the
// completion percentage — is measured against this.
Reserved int64 `json:"reserved" api:"required"`
// Whether `reserved` is an upper bound the batch may finish under. True only for a
// crawl that follows links, whose reachable page count is unknowable until it
// runs. False for a scrape and for a sitemap crawl, where `reserved` is an exact
// page count.
ReservedIsCeiling bool `json:"reserved_is_ceiling" api:"required"`
// URLs in the list you sent, before validation and de-duplication. Null for a
// crawl, which is given a source rather than a list.
Submitted int64 `json:"submitted" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
Duplicates respjson.Field
Invalid respjson.Field
Reserved respjson.Field
ReservedIsCeiling respjson.Field
Submitted respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r Intake) RawJSON() string { return r.JSON.raw }
func (r *Intake) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
type BatchGetResponse struct {
// Batch ID used to retrieve or cancel the job.
ID string `json:"id" api:"required"`
// The crawl controls as submitted, so the limits requested can be compared against
// what the crawl reached.
Crawl CrawlControls `json:"crawl" api:"required"`
// What this batch has done to your credit balance.
Credits BatchGetResponseCredits `json:"credits" api:"required"`
// A failure of the batch as a whole, distinct from the per-page failures in
// `page_errors`.
Failure Failure `json:"failure" api:"required"`
// What each page is returned as. Matches `input.data.format` on the submit
// request.
//
// Any of "markdown", "html".
Format BatchGetResponseFormat `json:"format" api:"required"`
// What submission took in, and what it charged for.
Input Intake `json:"input" api:"required"`
// Rejected URLs, up to 100. These are not charged.
InvalidURLs []BatchGetResponseInvalidURL `json:"invalid_urls" api:"required"`
// How pages were selected. Matches `input.mode` on the submit request.
//
// Any of "scrape", "crawl".
Mode BatchGetResponseMode `json:"mode" api:"required"`
// Individual page failures grouped by error code, sorted by count. Unrelated to
// `failure`, which is the batch itself failing.
PageErrors []PageErrorCount `json:"page_errors" api:"required"`
// Pages attempted so far. Use `status` to check completion.
Progress BatchGetResponseProgress `json:"progress" api:"required"`
// Download links, available once the batch reaches a final status and null before
// then. GET /batch/{batch_id}/results serves the same records as paginated JSON.
Results BatchGetResponseResults `json:"results" api:"required"`
// Current state. `completed`, `cancelled`, and `failed` are final.
//
// Any of "queued", "running", "cancelling", "completed", "cancelled", "failed".
Status BatchGetResponseStatus `json:"status" api:"required"`
// Tags stored on the batch at submission.
Tags []string `json:"tags" api:"required"`
Timing BatchGetResponseTiming `json:"timing" api:"required"`
// API key usage for this request.
KeyMetadata BatchGetResponseKeyMetadata `json:"key_metadata"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
ID respjson.Field
Crawl respjson.Field
Credits respjson.Field
Failure respjson.Field
Format respjson.Field
Input respjson.Field
InvalidURLs respjson.Field
Mode respjson.Field
PageErrors respjson.Field
Progress respjson.Field
Results respjson.Field
Status respjson.Field
Tags respjson.Field
Timing respjson.Field
KeyMetadata respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchGetResponse) RawJSON() string { return r.JSON.raw }
func (r *BatchGetResponse) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// What this batch has done to your credit balance.
type BatchGetResponseCredits struct {
// `reserved` minus `refunded` — what the batch has cost so far. Equal to
// `reserved` until the batch settles.
Net int64 `json:"net" api:"required"`
// Credits returned for pages that did not succeed. Stays 0 until the batch reaches
// a final status, then settles in one movement.
Refunded int64 `json:"refunded" api:"required"`
// Credits debited from your balance the moment the batch was accepted. This is a
// charge, not a forecast — the whole amount leaves the balance up front.
Reserved int64 `json:"reserved" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
Net respjson.Field
Refunded respjson.Field
Reserved respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchGetResponseCredits) RawJSON() string { return r.JSON.raw }
func (r *BatchGetResponseCredits) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// What each page is returned as. Matches `input.data.format` on the submit
// request.
type BatchGetResponseFormat string
const (
BatchGetResponseFormatMarkdown BatchGetResponseFormat = "markdown"
BatchGetResponseFormatHTML BatchGetResponseFormat = "html"
)
type BatchGetResponseInvalidURL struct {
// Why it was rejected.
Reason string `json:"reason" api:"required"`
// Rejected URL.
URL string `json:"url" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
Reason respjson.Field
URL respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchGetResponseInvalidURL) RawJSON() string { return r.JSON.raw }
func (r *BatchGetResponseInvalidURL) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// How pages were selected. Matches `input.mode` on the submit request.
type BatchGetResponseMode string
const (
BatchGetResponseModeScrape BatchGetResponseMode = "scrape"
BatchGetResponseModeCrawl BatchGetResponseMode = "crawl"
)
// Pages attempted so far. Use `status` to check completion.
type BatchGetResponseProgress struct {
// Pages that could not be scraped.
Failed int64 `json:"failed" api:"required"`
// Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it
// never reached; a crawl whose `input.reserved_is_ceiling` is true reports 0 once
// final, because its unspent budget was never real pages.
Pending int64 `json:"pending" api:"required"`
// Pages scraped successfully.
Succeeded int64 `json:"succeeded" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
Failed respjson.Field
Pending respjson.Field
Succeeded respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchGetResponseProgress) RawJSON() string { return r.JSON.raw }
func (r *BatchGetResponseProgress) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// Download links, available once the batch reaches a final status and null before
// then. GET /batch/{batch_id}/results serves the same records as paginated JSON.
type BatchGetResponseResults struct {
// When the download URLs expire.
ExpiresAt string `json:"expires_at" api:"required"`
// Result files. Order is not guaranteed.
Files []BatchGetResponseResultsFile `json:"files" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
ExpiresAt respjson.Field
Files respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchGetResponseResults) RawJSON() string { return r.JSON.raw }
func (r *BatchGetResponseResults) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
type BatchGetResponseResultsFile struct {
// Compressed file size in bytes.
Bytes int64 `json:"bytes" api:"required"`
// Results in this file.
Items int64 `json:"items" api:"required"`
// Temporary URL for a gzipped NDJSON file.
URL string `json:"url" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
Bytes respjson.Field
Items respjson.Field
URL respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchGetResponseResultsFile) RawJSON() string { return r.JSON.raw }
func (r *BatchGetResponseResultsFile) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// Current state. `completed`, `cancelled`, and `failed` are final.
type BatchGetResponseStatus string
const (
BatchGetResponseStatusQueued BatchGetResponseStatus = "queued"
BatchGetResponseStatusRunning BatchGetResponseStatus = "running"
BatchGetResponseStatusCancelling BatchGetResponseStatus = "cancelling"
BatchGetResponseStatusCompleted BatchGetResponseStatus = "completed"
BatchGetResponseStatusCancelled BatchGetResponseStatus = "cancelled"
BatchGetResponseStatusFailed BatchGetResponseStatus = "failed"
)
type BatchGetResponseTiming struct {
// When processing finished. Null while active.
CompletedAt string `json:"completed_at" api:"required"`
// When the batch was created.
CreatedAt string `json:"created_at" api:"required"`
// When processing started. Null while queued.
StartedAt string `json:"started_at" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
CompletedAt respjson.Field
CreatedAt respjson.Field
StartedAt respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchGetResponseTiming) RawJSON() string { return r.JSON.raw }
func (r *BatchGetResponseTiming) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// API key usage for this request.
type BatchGetResponseKeyMetadata struct {
// The number of credits consumed by this request.
CreditsConsumed int64 `json:"credits_consumed" api:"required"`
// The number of credits remaining for your organization after this request.
CreditsRemaining int64 `json:"credits_remaining" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
CreditsConsumed respjson.Field
CreditsRemaining respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchGetResponseKeyMetadata) RawJSON() string { return r.JSON.raw }
func (r *BatchGetResponseKeyMetadata) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
type BatchListResponse struct {
// Batches on this page.
Data []BatchListResponseData `json:"data"`
// Whether another page is available.
HasMore bool `json:"has_more"`
// Metadata about the API key used for the request. Included in every response
// whenever a valid API key is provided, even when the response status is not 200.
KeyMetadata BatchListResponseKeyMetadata `json:"key_metadata"`
// Cursor for the next page.
NextCursor string `json:"next_cursor" api:"nullable"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
Data respjson.Field
HasMore respjson.Field
KeyMetadata respjson.Field
NextCursor respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchListResponse) RawJSON() string { return r.JSON.raw }
func (r *BatchListResponse) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// An asynchronous web scraping job.
type BatchListResponseData struct {
// Batch ID used to retrieve or cancel the job.
ID string `json:"id" api:"required"`
// The crawl controls as submitted, so the limits requested can be compared against
// what the crawl reached.
Crawl CrawlControls `json:"crawl" api:"required"`
// What this batch has done to your credit balance.
Credits BatchListResponseDataCredits `json:"credits" api:"required"`
// A failure of the batch as a whole, distinct from the per-page failures in
// `page_errors`.
Failure Failure `json:"failure" api:"required"`
// What each page is returned as. Matches `input.data.format` on the submit
// request.
//
// Any of "markdown", "html".
Format string `json:"format" api:"required"`
// What submission took in, and what it charged for.
Input Intake `json:"input" api:"required"`
// How pages were selected. Matches `input.mode` on the submit request.
//
// Any of "scrape", "crawl".
Mode string `json:"mode" api:"required"`
// Individual page failures grouped by error code, sorted by count. Unrelated to
// `failure`, which is the batch itself failing.
PageErrors []PageErrorCount `json:"page_errors" api:"required"`
// Pages attempted so far. Use `status` to check completion.
Progress BatchListResponseDataProgress `json:"progress" api:"required"`
// Download links, available once the batch reaches a final status and null before
// then. GET /batch/{batch_id}/results serves the same records as paginated JSON.
Results BatchListResponseDataResults `json:"results" api:"required"`
// Current state. `completed`, `cancelled`, and `failed` are final.
//
// Any of "queued", "running", "cancelling", "completed", "cancelled", "failed".
Status string `json:"status" api:"required"`
// Tags stored on the batch at submission.
Tags []string `json:"tags" api:"required"`
Timing BatchListResponseDataTiming `json:"timing" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
ID respjson.Field
Crawl respjson.Field
Credits respjson.Field
Failure respjson.Field
Format respjson.Field
Input respjson.Field
Mode respjson.Field
PageErrors respjson.Field
Progress respjson.Field
Results respjson.Field
Status respjson.Field
Tags respjson.Field
Timing respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchListResponseData) RawJSON() string { return r.JSON.raw }
func (r *BatchListResponseData) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// What this batch has done to your credit balance.
type BatchListResponseDataCredits struct {
// `reserved` minus `refunded` — what the batch has cost so far. Equal to
// `reserved` until the batch settles.
Net int64 `json:"net" api:"required"`
// Credits returned for pages that did not succeed. Stays 0 until the batch reaches
// a final status, then settles in one movement.
Refunded int64 `json:"refunded" api:"required"`
// Credits debited from your balance the moment the batch was accepted. This is a
// charge, not a forecast — the whole amount leaves the balance up front.
Reserved int64 `json:"reserved" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
Net respjson.Field
Refunded respjson.Field
Reserved respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchListResponseDataCredits) RawJSON() string { return r.JSON.raw }
func (r *BatchListResponseDataCredits) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// Pages attempted so far. Use `status` to check completion.
type BatchListResponseDataProgress struct {
// Pages that could not be scraped.
Failed int64 `json:"failed" api:"required"`
// Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it
// never reached; a crawl whose `input.reserved_is_ceiling` is true reports 0 once
// final, because its unspent budget was never real pages.
Pending int64 `json:"pending" api:"required"`
// Pages scraped successfully.
Succeeded int64 `json:"succeeded" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
Failed respjson.Field
Pending respjson.Field
Succeeded respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchListResponseDataProgress) RawJSON() string { return r.JSON.raw }
func (r *BatchListResponseDataProgress) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// Download links, available once the batch reaches a final status and null before
// then. GET /batch/{batch_id}/results serves the same records as paginated JSON.
type BatchListResponseDataResults struct {
// When the download URLs expire.
ExpiresAt string `json:"expires_at" api:"required"`
// Result files. Order is not guaranteed.
Files []BatchListResponseDataResultsFile `json:"files" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
ExpiresAt respjson.Field
Files respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchListResponseDataResults) RawJSON() string { return r.JSON.raw }
func (r *BatchListResponseDataResults) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
type BatchListResponseDataResultsFile struct {
// Compressed file size in bytes.
Bytes int64 `json:"bytes" api:"required"`
// Results in this file.
Items int64 `json:"items" api:"required"`
// Temporary URL for a gzipped NDJSON file.
URL string `json:"url" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
Bytes respjson.Field
Items respjson.Field
URL respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchListResponseDataResultsFile) RawJSON() string { return r.JSON.raw }
func (r *BatchListResponseDataResultsFile) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
type BatchListResponseDataTiming struct {
// When processing finished. Null while active.
CompletedAt string `json:"completed_at" api:"required"`
// When the batch was created.
CreatedAt string `json:"created_at" api:"required"`
// When processing started. Null while queued.
StartedAt string `json:"started_at" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
CompletedAt respjson.Field
CreatedAt respjson.Field
StartedAt respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchListResponseDataTiming) RawJSON() string { return r.JSON.raw }
func (r *BatchListResponseDataTiming) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// Metadata about the API key used for the request. Included in every response
// whenever a valid API key is provided, even when the response status is not 200.
type BatchListResponseKeyMetadata struct {
// The number of credits consumed by this request.
CreditsConsumed int64 `json:"credits_consumed" api:"required"`
// The number of credits remaining for your organization after this request.
CreditsRemaining int64 `json:"credits_remaining" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
CreditsConsumed respjson.Field
CreditsRemaining respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchListResponseKeyMetadata) RawJSON() string { return r.JSON.raw }
func (r *BatchListResponseKeyMetadata) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
type BatchCancelResponse struct {
// Batch ID.
ID string `json:"id" api:"required"`
// The crawl controls as submitted, so the limits requested can be compared against
// what the crawl reached.
Crawl CrawlControls `json:"crawl" api:"required"`
// What this batch cost so far.
Credits BatchCancelResponseCredits `json:"credits" api:"required"`
// What each page is returned as.
//
// Any of "markdown", "html".
Format BatchCancelResponseFormat `json:"format" api:"required"`
// What submission took in, and what it charged for.
Input Intake `json:"input" api:"required"`
// How pages were selected.
//
// Any of "scrape", "crawl".
Mode BatchCancelResponseMode `json:"mode" api:"required"`
// Page failures so far, grouped by error code and sorted by count.
PageErrors []PageErrorCount `json:"page_errors" api:"required"`
// How far the batch got before cancellation.
Progress BatchCancelResponseProgress `json:"progress" api:"required"`
// Always `cancelling`. Work already in flight finishes; the batch reaches
// `cancelled` shortly after.
//
// Any of "cancelling".
Status BatchCancelResponseStatus `json:"status" api:"required"`
// Tags stored on the batch at submission.
Tags []string `json:"tags" api:"required"`
// There is no finish time yet — the batch is still winding down.
Timing BatchCancelResponseTiming `json:"timing" api:"required"`
// API key usage for this request.
KeyMetadata BatchCancelResponseKeyMetadata `json:"key_metadata"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
ID respjson.Field
Crawl respjson.Field
Credits respjson.Field
Format respjson.Field
Input respjson.Field
Mode respjson.Field
PageErrors respjson.Field
Progress respjson.Field
Status respjson.Field
Tags respjson.Field
Timing respjson.Field
KeyMetadata respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchCancelResponse) RawJSON() string { return r.JSON.raw }
func (r *BatchCancelResponse) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// What this batch cost so far.
type BatchCancelResponseCredits struct {
// Credits debited at submission. The unspent remainder is refunded once the batch
// settles — read `credits.refunded` from GET /batch/{batch_id} then.
Reserved int64 `json:"reserved" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
Reserved respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchCancelResponseCredits) RawJSON() string { return r.JSON.raw }
func (r *BatchCancelResponseCredits) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// What each page is returned as.
type BatchCancelResponseFormat string
const (
BatchCancelResponseFormatMarkdown BatchCancelResponseFormat = "markdown"
BatchCancelResponseFormatHTML BatchCancelResponseFormat = "html"
)
// How pages were selected.
type BatchCancelResponseMode string
const (
BatchCancelResponseModeScrape BatchCancelResponseMode = "scrape"
BatchCancelResponseModeCrawl BatchCancelResponseMode = "crawl"
)
// How far the batch got before cancellation.
type BatchCancelResponseProgress struct {
// Pages that could not be scraped before the request landed.
Failed int64 `json:"failed" api:"required"`
// Reserved pages that will now be skipped, and refunded when the batch settles.
Pending int64 `json:"pending" api:"required"`
// Pages scraped successfully before the request landed.
Succeeded int64 `json:"succeeded" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
Failed respjson.Field
Pending respjson.Field
Succeeded respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchCancelResponseProgress) RawJSON() string { return r.JSON.raw }
func (r *BatchCancelResponseProgress) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// Always `cancelling`. Work already in flight finishes; the batch reaches
// `cancelled` shortly after.
type BatchCancelResponseStatus string
const (
BatchCancelResponseStatusCancelling BatchCancelResponseStatus = "cancelling"
)
// There is no finish time yet — the batch is still winding down.
type BatchCancelResponseTiming struct {
// When the batch was created.
CreatedAt string `json:"created_at" api:"required"`
// When processing started. Null if it was cancelled while still queued.
StartedAt string `json:"started_at" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
CreatedAt respjson.Field
StartedAt respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchCancelResponseTiming) RawJSON() string { return r.JSON.raw }
func (r *BatchCancelResponseTiming) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// API key usage for this request.
type BatchCancelResponseKeyMetadata struct {
// The number of credits consumed by this request.
CreditsConsumed int64 `json:"credits_consumed" api:"required"`
// The number of credits remaining for your organization after this request.
CreditsRemaining int64 `json:"credits_remaining" api:"required"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
CreditsConsumed respjson.Field
CreditsRemaining respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchCancelResponseKeyMetadata) RawJSON() string { return r.JSON.raw }
func (r *BatchCancelResponseKeyMetadata) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
type BatchGetResultsResponse struct {
// Result records on this page.
Data []BatchGetResultsResponseDataUnion `json:"data"`
// Whether another page is available.
HasMore bool `json:"has_more"`
// Metadata about the API key used for the request. Included in every response
// whenever a valid API key is provided, even when the response status is not 200.
KeyMetadata BatchGetResultsResponseKeyMetadata `json:"key_metadata"`
// Cursor for the next page.
NextCursor string `json:"next_cursor" api:"nullable"`
// JSON contains metadata for fields, check presence with [respjson.Field.Valid].
JSON struct {
Data respjson.Field
HasMore respjson.Field
KeyMetadata respjson.Field
NextCursor respjson.Field
ExtraFields map[string]respjson.Field
raw string
} `json:"-"`
}
// Returns the unmodified JSON received from the API
func (r BatchGetResultsResponse) RawJSON() string { return r.JSON.raw }
func (r *BatchGetResultsResponse) UnmarshalJSON(data []byte) error {
return apijson.UnmarshalRoot(data, r)
}
// BatchGetResultsResponseDataUnion contains all possible properties and values
// from [BatchGetResultsResponseDataOk], [BatchGetResultsResponseDataError].
//
// Use the [BatchGetResultsResponseDataUnion.AsAny] method to switch on the
// variant.
//
// Use the methods beginning with 'As' to cast the union to one of its variants.
type BatchGetResultsResponseDataUnion struct {
// This field is from variant [BatchGetResultsResponseDataOk].
FinalURL string `json:"final_url"`
// This field is from variant [BatchGetResultsResponseDataOk].
HTTPStatus int64 `json:"http_status"`
// This field is from variant [BatchGetResultsResponseDataOk].
Metadata BatchGetResultsResponseDataOkMetadata `json:"metadata"`
// Any of "ok", "error".
Status string `json:"status"`
URL string `json:"url"`
// This field is from variant [BatchGetResultsResponseDataOk].
HTML string `json:"html"`
ItemID string `json:"itemId"`
// This field is from variant [BatchGetResultsResponseDataOk].
Markdown string `json:"markdown"`
Meta any `json:"meta"`
// This field is from variant [BatchGetResultsResponseDataError].
ErrorCode string `json:"error_code"`
// This field is from variant [BatchGetResultsResponseDataError].