1use crate::shader::constants::{
2 MAX_BUCKET_SIZE, MAX_TABLE_SIZE, NUM_BUCKETS, NUM_MATCH_BUCKETS, NUM_S_BUCKETS,
3 REDUCED_MATCHES_COUNT,
4};
5use crate::shader::find_matches_and_compute_f7::{NUM_ELEMENTS_PER_S_BUCKET, ProofTargets};
6use crate::shader::find_proofs::ProofsHost;
7use crate::shader::types::{Metadata, Position, PositionR};
8use crate::shader::{compute_f1, find_proofs, select_shader_features_limits};
9use ab_chacha8::{ChaCha8Block, ChaCha8State, block_to_bytes};
10use futures::stream::FuturesOrdered;
11use futures::{StreamExt, TryStreamExt};
12use parking_lot::Mutex;
13use rclite::Arc;
14use std::num::NonZeroU8;
15use std::{fmt, iter};
16use subspace_core_primitives::pos::PosSeed;
17use wgpu::{
18 AdapterInfo, Backend, BackendOptions, Backends, BindGroup, BindGroupDescriptor, BindGroupEntry,
19 BindGroupLayoutDescriptor, BindGroupLayoutEntry, BindingType, Buffer, BufferAddress,
20 BufferAsyncError, BufferBindingType, BufferDescriptor, BufferUsages, CommandEncoderDescriptor,
21 ComputePassDescriptor, ComputePipeline, ComputePipelineDescriptor, DeviceDescriptor,
22 DeviceType, Instance, InstanceDescriptor, InstanceFlags, MapMode, MapRangeError,
23 MemoryBudgetThresholds, PipelineCompilationOptions, PipelineLayoutDescriptor, PollError,
24 PollType, Queue, RequestDeviceError, ShaderModule, ShaderRuntimeChecks, ShaderStages,
25};
26
27#[derive(Debug, thiserror::Error)]
29pub enum RecordEncodingError {
30 #[error("Proof creation failed previously and the device is now considered broken")]
32 DeviceBroken,
33 #[error("Failed to map buffer: {0}")]
35 BufferMapping(#[from] BufferAsyncError),
36 #[error("Failed to get mapped buffer range: {0}")]
38 MapRangeError(#[from] MapRangeError),
39 #[error("Poll error: {0}")]
41 DevicePoll(#[from] PollError),
42}
43
44pub struct ProofsHostWrapper<'a> {
46 proofs: &'a ProofsHost,
47 proofs_host: &'a Buffer,
48}
49
50impl ProofsHostWrapper<'_> {
51 pub fn proofs(&self) -> &ProofsHost {
53 self.proofs
54 }
55}
56
57impl fmt::Debug for ProofsHostWrapper<'_> {
58 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
59 f.debug_struct("ProofsHostWrapper").finish_non_exhaustive()
60 }
61}
62
63impl Drop for ProofsHostWrapper<'_> {
64 fn drop(&mut self) {
65 self.proofs_host.unmap();
66 }
67}
68
69#[derive(Clone)]
71pub struct Device {
72 id: u32,
73 devices: Vec<(wgpu::Device, Queue, ShaderModule)>,
74 adapter_info: AdapterInfo,
75}
76
77impl fmt::Debug for Device {
78 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
79 f.debug_struct("Device")
80 .field("id", &self.id)
81 .field("name", &self.adapter_info.name)
82 .field("device_type", &self.adapter_info.device_type)
83 .field("driver", &self.adapter_info.driver)
84 .field("driver_info", &self.adapter_info.driver_info)
85 .field("backend", &self.adapter_info.backend)
86 .finish_non_exhaustive()
87 }
88}
89
90impl Device {
91 pub async fn enumerate<NOQ>(number_of_queues: NOQ) -> Vec<Self>
93 where
94 NOQ: Fn(DeviceType) -> NonZeroU8,
95 {
96 let backends = Backends::from_env().unwrap_or(Backends::METAL | Backends::VULKAN);
97 let instance = Instance::new(InstanceDescriptor {
98 backends,
99 flags: if cfg!(debug_assertions) {
100 InstanceFlags::debugging().with_env()
101 } else {
102 InstanceFlags::from_env_or_default()
103 },
104 memory_budget_thresholds: MemoryBudgetThresholds::default(),
105 backend_options: BackendOptions::from_env_or_default(),
106 display: None,
107 });
108
109 let adapters = instance.enumerate_adapters(backends).await;
110 let number_of_queues = &number_of_queues;
111
112 adapters
113 .into_iter()
114 .zip(0..)
115 .map(|(adapter, id)| async move {
116 let adapter_info = adapter.get_info();
117
118 #[expect(
119 clippy::manual_let_else,
120 reason = "https://github.com/rust-lang/rust/issues/157152"
121 )]
122 let (shader, required_features, required_limits) =
123 if let Some((shader, required_features, required_limits)) =
124 select_shader_features_limits(&adapter)
125 {
126 (shader, required_features, required_limits)
135 } else {
136 return None;
145 };
146
147 let devices = iter::repeat_with(|| async {
150 let (device, queue) = adapter
151 .request_device(&DeviceDescriptor {
152 label: None,
153 required_features,
154 required_limits: required_limits.clone(),
155 ..DeviceDescriptor::default()
156 })
157 .await
158 .inspect_err(|_error| {
159 })?;
163 let module = if cfg!(debug_assertions) {
164 device.create_shader_module(shader.clone())
165 } else {
166 unsafe {
170 device.create_shader_module_trusted(
171 shader.clone(),
172 ShaderRuntimeChecks::unchecked(),
173 )
174 }
175 };
176
177 Ok::<_, RequestDeviceError>((device, queue, module))
178 })
179 .take(usize::from(
180 number_of_queues(adapter_info.device_type).get(),
181 ))
182 .collect::<FuturesOrdered<_>>()
183 .try_collect::<Vec<_>>()
184 .await
185 .ok()?;
186
187 Some(Self {
188 id,
189 devices,
190 adapter_info,
191 })
192 })
193 .collect::<FuturesOrdered<_>>()
194 .filter_map(|device| async move { device })
195 .collect()
196 .await
197 }
198
199 pub fn id(&self) -> u32 {
201 self.id
202 }
203
204 pub fn name(&self) -> &str {
206 &self.adapter_info.name
207 }
208
209 pub fn device_type(&self) -> DeviceType {
211 self.adapter_info.device_type
212 }
213
214 pub fn driver(&self) -> &str {
216 &self.adapter_info.driver
217 }
218
219 pub fn driver_info(&self) -> &str {
221 &self.adapter_info.driver_info
222 }
223
224 pub fn backend(&self) -> Backend {
226 self.adapter_info.backend
227 }
228
229 pub fn create_proofs_encoder_instances(&self) -> Vec<GpuRecordsEncoderInstance> {
232 self.devices
233 .clone()
234 .into_iter()
235 .map(|(device, queue, module)| GpuRecordsEncoderInstance::new(device, queue, module))
236 .collect()
237 }
238}
239
240pub struct GpuRecordsEncoderInstance {
241 device: wgpu::Device,
242 queue: Queue,
243 mapping_error: Arc<Mutex<Option<BufferAsyncError>>>,
244 tainted: bool,
245 initial_state_host: Buffer,
246 initial_state_gpu: Buffer,
247 proofs_host: Buffer,
248 proofs_gpu: Buffer,
249 bind_group_compute_f1: BindGroup,
250 compute_pipeline_compute_f1: ComputePipeline,
251 bind_group_sort_buckets_a: BindGroup,
252 compute_pipeline_sort_buckets_a: ComputePipeline,
253 bind_group_sort_buckets_b: BindGroup,
254 compute_pipeline_sort_buckets_b: ComputePipeline,
255 bind_group_find_matches_and_compute_f2: BindGroup,
256 compute_pipeline_find_matches_and_compute_f2: ComputePipeline,
257 bind_group_find_matches_and_compute_f3: BindGroup,
258 compute_pipeline_find_matches_and_compute_f3: ComputePipeline,
259 bind_group_find_matches_and_compute_f4: BindGroup,
260 compute_pipeline_find_matches_and_compute_f4: ComputePipeline,
261 bind_group_find_matches_and_compute_f5: BindGroup,
262 compute_pipeline_find_matches_and_compute_f5: ComputePipeline,
263 bind_group_find_matches_and_compute_f6: BindGroup,
264 compute_pipeline_find_matches_and_compute_f6: ComputePipeline,
265 bind_group_find_matches_and_compute_f7: BindGroup,
266 compute_pipeline_find_matches_and_compute_f7: ComputePipeline,
267 bind_group_find_proofs: BindGroup,
268 compute_pipeline_find_proofs: ComputePipeline,
269}
270
271impl fmt::Debug for GpuRecordsEncoderInstance {
272 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
273 f.debug_struct("GpuRecordsEncoderInstance")
274 .finish_non_exhaustive()
275 }
276}
277
278impl GpuRecordsEncoderInstance {
279 fn new(device: wgpu::Device, queue: Queue, module: ShaderModule) -> Self {
280 let initial_state_host = device.create_buffer(&BufferDescriptor {
281 label: Some("initial_state_host"),
282 size: size_of::<ChaCha8Block>() as BufferAddress,
283 usage: BufferUsages::MAP_WRITE | BufferUsages::COPY_SRC,
284 mapped_at_creation: true,
285 });
286
287 let initial_state_gpu = device.create_buffer(&BufferDescriptor {
288 label: Some("initial_state_gpu"),
289 size: initial_state_host.size(),
290 usage: BufferUsages::UNIFORM | BufferUsages::COPY_DST,
291 mapped_at_creation: false,
292 });
293
294 let bucket_sizes_gpu_buffer_size = size_of::<[u32; NUM_BUCKETS]>() as BufferAddress;
295 let table_6_proof_targets_sizes_gpu_buffer_size =
296 size_of::<[u32; NUM_S_BUCKETS]>() as BufferAddress;
297 let bucket_sizes_gpu = device.create_buffer(&BufferDescriptor {
301 label: Some("bucket_sizes_gpu"),
302 size: bucket_sizes_gpu_buffer_size.max(table_6_proof_targets_sizes_gpu_buffer_size),
303 usage: BufferUsages::STORAGE,
304 mapped_at_creation: false,
305 });
306 let table_6_proof_targets_sizes_gpu = bucket_sizes_gpu.clone();
308
309 let buckets_a_gpu = device.create_buffer(&BufferDescriptor {
310 label: Some("buckets_a_gpu"),
311 size: size_of::<[[PositionR; MAX_BUCKET_SIZE]; NUM_BUCKETS]>() as BufferAddress,
312 usage: BufferUsages::STORAGE,
313 mapped_at_creation: false,
314 });
315
316 let buckets_b_gpu = device.create_buffer(&BufferDescriptor {
317 label: Some("buckets_b_gpu"),
318 size: buckets_a_gpu.size(),
319 usage: BufferUsages::STORAGE,
320 mapped_at_creation: false,
321 });
322
323 let positions_f2_gpu = device.create_buffer(&BufferDescriptor {
324 label: Some("positions_f2_gpu"),
325 size: size_of::<[[[Position; 2]; REDUCED_MATCHES_COUNT]; NUM_MATCH_BUCKETS]>()
326 as BufferAddress,
327 usage: BufferUsages::STORAGE,
328 mapped_at_creation: false,
329 });
330
331 let positions_f3_gpu = device.create_buffer(&BufferDescriptor {
332 label: Some("positions_f3_gpu"),
333 size: positions_f2_gpu.size(),
334 usage: BufferUsages::STORAGE,
335 mapped_at_creation: false,
336 });
337
338 let positions_f4_gpu = device.create_buffer(&BufferDescriptor {
339 label: Some("positions_f4_gpu"),
340 size: positions_f2_gpu.size(),
341 usage: BufferUsages::STORAGE,
342 mapped_at_creation: false,
343 });
344
345 let positions_f5_gpu = device.create_buffer(&BufferDescriptor {
346 label: Some("positions_f5_gpu"),
347 size: positions_f2_gpu.size(),
348 usage: BufferUsages::STORAGE,
349 mapped_at_creation: false,
350 });
351
352 let positions_f6_gpu = device.create_buffer(&BufferDescriptor {
353 label: Some("positions_f6_gpu"),
354 size: positions_f2_gpu.size(),
355 usage: BufferUsages::STORAGE,
356 mapped_at_creation: false,
357 });
358
359 let metadatas_gpu_buffer_size =
360 size_of::<[[Metadata; REDUCED_MATCHES_COUNT]; NUM_MATCH_BUCKETS]>() as BufferAddress;
361 let table_6_proof_targets_gpu_buffer_size = size_of::<
362 [[ProofTargets; NUM_ELEMENTS_PER_S_BUCKET]; NUM_S_BUCKETS],
363 >() as BufferAddress;
364 let metadatas_a_gpu = device.create_buffer(&BufferDescriptor {
365 label: Some("metadatas_a_gpu"),
366 size: metadatas_gpu_buffer_size.max(table_6_proof_targets_gpu_buffer_size),
367 usage: BufferUsages::STORAGE,
368 mapped_at_creation: false,
369 });
370 let table_6_proof_targets_gpu = metadatas_a_gpu.clone();
372
373 let metadatas_b_gpu = device.create_buffer(&BufferDescriptor {
374 label: Some("metadatas_b_gpu"),
375 size: metadatas_gpu_buffer_size,
376 usage: BufferUsages::STORAGE,
377 mapped_at_creation: false,
378 });
379
380 let proofs_host = device.create_buffer(&BufferDescriptor {
381 label: Some("proofs_host"),
382 size: size_of::<ProofsHost>() as BufferAddress,
383 usage: BufferUsages::MAP_READ | BufferUsages::COPY_DST,
384 mapped_at_creation: false,
385 });
386
387 let proofs_gpu = device.create_buffer(&BufferDescriptor {
388 label: Some("proofs_gpu"),
389 size: proofs_host.size(),
390 usage: BufferUsages::STORAGE | BufferUsages::COPY_SRC,
391 mapped_at_creation: false,
392 });
393
394 let (bind_group_compute_f1, compute_pipeline_compute_f1) =
395 bind_group_and_pipeline_compute_f1(
396 &device,
397 &module,
398 &initial_state_gpu,
399 &bucket_sizes_gpu,
400 &buckets_a_gpu,
401 );
402 let (bind_group_sort_buckets_a, compute_pipeline_sort_buckets_a) =
403 bind_group_and_pipeline_sort_buckets(
404 &device,
405 &module,
406 &bucket_sizes_gpu,
407 &buckets_a_gpu,
408 );
409
410 let (bind_group_sort_buckets_b, compute_pipeline_sort_buckets_b) =
411 bind_group_and_pipeline_sort_buckets(
412 &device,
413 &module,
414 &bucket_sizes_gpu,
415 &buckets_b_gpu,
416 );
417
418 let (bind_group_find_matches_and_compute_f2, compute_pipeline_find_matches_and_compute_f2) =
419 bind_group_and_pipeline_find_matches_and_compute_f2(
420 &device,
421 &module,
422 &buckets_a_gpu,
423 &bucket_sizes_gpu,
424 &buckets_b_gpu,
425 &positions_f2_gpu,
426 &metadatas_b_gpu,
427 );
428
429 let (bind_group_find_matches_and_compute_f3, compute_pipeline_find_matches_and_compute_f3) =
430 bind_group_and_pipeline_find_matches_and_compute_fn::<3>(
431 &device,
432 &module,
433 &buckets_b_gpu,
434 &metadatas_b_gpu,
435 &bucket_sizes_gpu,
436 &buckets_a_gpu,
437 &positions_f3_gpu,
438 &metadatas_a_gpu,
439 );
440
441 let (bind_group_find_matches_and_compute_f4, compute_pipeline_find_matches_and_compute_f4) =
442 bind_group_and_pipeline_find_matches_and_compute_fn::<4>(
443 &device,
444 &module,
445 &buckets_a_gpu,
446 &metadatas_a_gpu,
447 &bucket_sizes_gpu,
448 &buckets_b_gpu,
449 &positions_f4_gpu,
450 &metadatas_b_gpu,
451 );
452
453 let (bind_group_find_matches_and_compute_f5, compute_pipeline_find_matches_and_compute_f5) =
454 bind_group_and_pipeline_find_matches_and_compute_fn::<5>(
455 &device,
456 &module,
457 &buckets_b_gpu,
458 &metadatas_b_gpu,
459 &bucket_sizes_gpu,
460 &buckets_a_gpu,
461 &positions_f5_gpu,
462 &metadatas_a_gpu,
463 );
464
465 let (bind_group_find_matches_and_compute_f6, compute_pipeline_find_matches_and_compute_f6) =
466 bind_group_and_pipeline_find_matches_and_compute_fn::<6>(
467 &device,
468 &module,
469 &buckets_a_gpu,
470 &metadatas_a_gpu,
471 &bucket_sizes_gpu,
472 &buckets_b_gpu,
473 &positions_f6_gpu,
474 &metadatas_b_gpu,
475 );
476
477 let (bind_group_find_matches_and_compute_f7, compute_pipeline_find_matches_and_compute_f7) =
478 bind_group_and_pipeline_find_matches_and_compute_f7(
479 &device,
480 &module,
481 &buckets_b_gpu,
482 &metadatas_b_gpu,
483 &table_6_proof_targets_sizes_gpu,
484 &table_6_proof_targets_gpu,
485 );
486
487 let (bind_group_find_proofs, compute_pipeline_find_proofs) =
488 bind_group_and_pipeline_find_proofs(
489 &device,
490 &module,
491 &positions_f2_gpu,
492 &positions_f3_gpu,
493 &positions_f4_gpu,
494 &positions_f5_gpu,
495 &positions_f6_gpu,
496 &table_6_proof_targets_sizes_gpu,
497 &table_6_proof_targets_gpu,
498 &proofs_gpu,
499 );
500
501 Self {
502 device,
503 queue,
504 mapping_error: Arc::new(Mutex::new(None)),
505 tainted: false,
506 initial_state_host,
507 initial_state_gpu,
508 proofs_host,
509 proofs_gpu,
510 bind_group_compute_f1,
511 compute_pipeline_compute_f1,
512 bind_group_sort_buckets_a,
513 compute_pipeline_sort_buckets_a,
514 bind_group_sort_buckets_b,
515 compute_pipeline_sort_buckets_b,
516 bind_group_find_matches_and_compute_f2,
517 compute_pipeline_find_matches_and_compute_f2,
518 bind_group_find_matches_and_compute_f3,
519 compute_pipeline_find_matches_and_compute_f3,
520 bind_group_find_matches_and_compute_f4,
521 compute_pipeline_find_matches_and_compute_f4,
522 bind_group_find_matches_and_compute_f5,
523 compute_pipeline_find_matches_and_compute_f5,
524 bind_group_find_matches_and_compute_f6,
525 compute_pipeline_find_matches_and_compute_f6,
526 bind_group_find_matches_and_compute_f7,
527 compute_pipeline_find_matches_and_compute_f7,
528 bind_group_find_proofs,
529 compute_pipeline_find_proofs,
530 }
531 }
532
533 pub fn create_proofs(
534 &mut self,
535 seed: &PosSeed,
536 ) -> Result<ProofsHostWrapper<'_>, RecordEncodingError> {
537 if self.tainted {
538 return Err(RecordEncodingError::DeviceBroken);
539 }
540 self.tainted = true;
541
542 let mut encoder = self
543 .device
544 .create_command_encoder(&CommandEncoderDescriptor {
545 label: Some("create_proofs"),
546 });
547
548 self.initial_state_host
550 .get_mapped_range_mut(..)?
551 .copy_from_slice(&block_to_bytes(
552 &ChaCha8State::init(seed, &[0; _]).to_repr(),
553 ));
554 self.initial_state_host.unmap();
555
556 encoder.copy_buffer_to_buffer(
557 &self.initial_state_host,
558 0,
559 &self.initial_state_gpu,
560 0,
561 self.initial_state_host.size(),
562 );
563
564 {
565 let mut cpass = encoder.begin_compute_pass(&ComputePassDescriptor {
566 label: Some("create_proofs"),
567 timestamp_writes: None,
568 });
569
570 cpass.set_bind_group(0, &self.bind_group_compute_f1, &[]);
571 cpass.set_pipeline(&self.compute_pipeline_compute_f1);
572 cpass.dispatch_workgroups(
573 MAX_TABLE_SIZE
574 .div_ceil(compute_f1::WORKGROUP_SIZE * compute_f1::ELEMENTS_PER_INVOCATION),
575 1,
576 1,
577 );
578
579 cpass.set_bind_group(0, &self.bind_group_sort_buckets_a, &[]);
580 cpass.set_pipeline(&self.compute_pipeline_sort_buckets_a);
581 cpass.dispatch_workgroups(NUM_BUCKETS as u32, 1, 1);
582
583 cpass.set_bind_group(0, &self.bind_group_find_matches_and_compute_f2, &[]);
584 cpass.set_pipeline(&self.compute_pipeline_find_matches_and_compute_f2);
585 cpass.dispatch_workgroups(NUM_MATCH_BUCKETS as u32, 1, 1);
586
587 cpass.set_bind_group(0, &self.bind_group_sort_buckets_b, &[]);
588 cpass.set_pipeline(&self.compute_pipeline_sort_buckets_b);
589 cpass.dispatch_workgroups(NUM_BUCKETS as u32, 1, 1);
590
591 cpass.set_bind_group(0, &self.bind_group_find_matches_and_compute_f3, &[]);
592 cpass.set_pipeline(&self.compute_pipeline_find_matches_and_compute_f3);
593 cpass.dispatch_workgroups(NUM_MATCH_BUCKETS as u32, 1, 1);
594
595 cpass.set_bind_group(0, &self.bind_group_sort_buckets_a, &[]);
596 cpass.set_pipeline(&self.compute_pipeline_sort_buckets_a);
597 cpass.dispatch_workgroups(NUM_BUCKETS as u32, 1, 1);
598
599 cpass.set_bind_group(0, &self.bind_group_find_matches_and_compute_f4, &[]);
600 cpass.set_pipeline(&self.compute_pipeline_find_matches_and_compute_f4);
601 cpass.dispatch_workgroups(NUM_MATCH_BUCKETS as u32, 1, 1);
602
603 cpass.set_bind_group(0, &self.bind_group_sort_buckets_b, &[]);
604 cpass.set_pipeline(&self.compute_pipeline_sort_buckets_b);
605 cpass.dispatch_workgroups(NUM_BUCKETS as u32, 1, 1);
606
607 cpass.set_bind_group(0, &self.bind_group_find_matches_and_compute_f5, &[]);
608 cpass.set_pipeline(&self.compute_pipeline_find_matches_and_compute_f5);
609 cpass.dispatch_workgroups(NUM_MATCH_BUCKETS as u32, 1, 1);
610
611 cpass.set_bind_group(0, &self.bind_group_sort_buckets_a, &[]);
612 cpass.set_pipeline(&self.compute_pipeline_sort_buckets_a);
613 cpass.dispatch_workgroups(NUM_BUCKETS as u32, 1, 1);
614
615 cpass.set_bind_group(0, &self.bind_group_find_matches_and_compute_f6, &[]);
616 cpass.set_pipeline(&self.compute_pipeline_find_matches_and_compute_f6);
617 cpass.dispatch_workgroups(NUM_MATCH_BUCKETS as u32, 1, 1);
618
619 cpass.set_bind_group(0, &self.bind_group_sort_buckets_b, &[]);
620 cpass.set_pipeline(&self.compute_pipeline_sort_buckets_b);
621 cpass.dispatch_workgroups(NUM_BUCKETS as u32, 1, 1);
622
623 cpass.set_bind_group(0, &self.bind_group_find_matches_and_compute_f7, &[]);
624 cpass.set_pipeline(&self.compute_pipeline_find_matches_and_compute_f7);
625 cpass.dispatch_workgroups(NUM_MATCH_BUCKETS as u32, 1, 1);
626
627 cpass.set_bind_group(0, &self.bind_group_find_proofs, &[]);
628 cpass.set_pipeline(&self.compute_pipeline_find_proofs);
629 cpass.dispatch_workgroups(NUM_S_BUCKETS as u32 / find_proofs::WORKGROUP_SIZE, 1, 1);
630 }
631
632 encoder.copy_buffer_to_buffer(
633 &self.proofs_gpu,
634 0,
635 &self.proofs_host,
636 0,
637 self.proofs_host.size(),
638 );
639
640 encoder.map_buffer_on_submit(&self.initial_state_host, MapMode::Write, .., {
642 let mapping_error = Arc::clone(&self.mapping_error);
643
644 move |r| {
645 if let Err(error) = r {
646 mapping_error.lock().replace(error);
647 }
648 }
649 });
650 encoder.map_buffer_on_submit(&self.proofs_host, MapMode::Read, .., {
651 let mapping_error = Arc::clone(&self.mapping_error);
652
653 move |r| {
654 if let Err(error) = r {
655 mapping_error.lock().replace(error);
656 }
657 }
658 });
659
660 let submission_index = self.queue.submit([encoder.finish()]);
661
662 self.device.poll(PollType::Wait {
663 submission_index: Some(submission_index),
664 timeout: None,
665 })?;
666
667 if let Some(error) = self.mapping_error.lock().take() {
668 return Err(RecordEncodingError::BufferMapping(error));
669 }
670
671 let proofs = {
672 let proofs_host_ptr = self
673 .proofs_host
674 .get_mapped_range(..)?
675 .as_ptr()
676 .cast::<ProofsHost>();
677 unsafe { &*proofs_host_ptr }
679 };
680
681 self.tainted = false;
682
683 Ok(ProofsHostWrapper {
684 proofs,
685 proofs_host: &self.proofs_host,
686 })
687 }
688}
689
690fn bind_group_and_pipeline_compute_f1(
691 device: &wgpu::Device,
692 module: &ShaderModule,
693 initial_state_gpu: &Buffer,
694 bucket_sizes_gpu: &Buffer,
695 buckets_gpu: &Buffer,
696) -> (BindGroup, ComputePipeline) {
697 let bind_group_layout = device.create_bind_group_layout(&BindGroupLayoutDescriptor {
698 label: Some("compute_f1"),
699 entries: &[
700 BindGroupLayoutEntry {
701 binding: 0,
702 count: None,
703 visibility: ShaderStages::COMPUTE,
704 ty: BindingType::Buffer {
705 has_dynamic_offset: false,
706 min_binding_size: None,
707 ty: BufferBindingType::Uniform,
708 },
709 },
710 BindGroupLayoutEntry {
711 binding: 1,
712 count: None,
713 visibility: ShaderStages::COMPUTE,
714 ty: BindingType::Buffer {
715 has_dynamic_offset: false,
716 min_binding_size: None,
717 ty: BufferBindingType::Storage { read_only: false },
718 },
719 },
720 BindGroupLayoutEntry {
721 binding: 2,
722 count: None,
723 visibility: ShaderStages::COMPUTE,
724 ty: BindingType::Buffer {
725 has_dynamic_offset: false,
726 min_binding_size: None,
727 ty: BufferBindingType::Storage { read_only: false },
728 },
729 },
730 ],
731 });
732
733 let pipeline_layout = device.create_pipeline_layout(&PipelineLayoutDescriptor {
734 label: Some("compute_f1"),
735 bind_group_layouts: &[Some(&bind_group_layout)],
736 immediate_size: 0,
737 });
738
739 let compute_pipeline = device.create_compute_pipeline(&ComputePipelineDescriptor {
740 compilation_options: PipelineCompilationOptions {
741 constants: &[],
742 zero_initialize_workgroup_memory: false,
743 },
744 cache: None,
745 label: Some("compute_f1"),
746 layout: Some(&pipeline_layout),
747 module,
748 entry_point: Some("compute_f1"),
749 });
750
751 let bind_group = device.create_bind_group(&BindGroupDescriptor {
752 label: Some("compute_f1"),
753 layout: &bind_group_layout,
754 entries: &[
755 BindGroupEntry {
756 binding: 0,
757 resource: initial_state_gpu.as_entire_binding(),
758 },
759 BindGroupEntry {
760 binding: 1,
761 resource: bucket_sizes_gpu.as_entire_binding(),
762 },
763 BindGroupEntry {
764 binding: 2,
765 resource: buckets_gpu.as_entire_binding(),
766 },
767 ],
768 });
769
770 (bind_group, compute_pipeline)
771}
772
773fn bind_group_and_pipeline_sort_buckets(
774 device: &wgpu::Device,
775 module: &ShaderModule,
776 bucket_sizes_gpu: &Buffer,
777 buckets_gpu: &Buffer,
778) -> (BindGroup, ComputePipeline) {
779 let bind_group_layout = device.create_bind_group_layout(&BindGroupLayoutDescriptor {
780 label: Some("sort_buckets"),
781 entries: &[
782 BindGroupLayoutEntry {
783 binding: 0,
784 count: None,
785 visibility: ShaderStages::COMPUTE,
786 ty: BindingType::Buffer {
787 has_dynamic_offset: false,
788 min_binding_size: None,
789 ty: BufferBindingType::Storage { read_only: false },
790 },
791 },
792 BindGroupLayoutEntry {
793 binding: 1,
794 count: None,
795 visibility: ShaderStages::COMPUTE,
796 ty: BindingType::Buffer {
797 has_dynamic_offset: false,
798 min_binding_size: None,
799 ty: BufferBindingType::Storage { read_only: false },
800 },
801 },
802 ],
803 });
804
805 let pipeline_layout = device.create_pipeline_layout(&PipelineLayoutDescriptor {
806 label: Some("sort_buckets"),
807 bind_group_layouts: &[Some(&bind_group_layout)],
808 immediate_size: 0,
809 });
810
811 let compute_pipeline = device.create_compute_pipeline(&ComputePipelineDescriptor {
812 compilation_options: PipelineCompilationOptions {
813 constants: &[],
814 zero_initialize_workgroup_memory: false,
815 },
816 cache: None,
817 label: Some("sort_buckets"),
818 layout: Some(&pipeline_layout),
819 module,
820 entry_point: Some("sort_buckets"),
821 });
822
823 let bind_group = device.create_bind_group(&BindGroupDescriptor {
824 label: Some("sort_buckets"),
825 layout: &bind_group_layout,
826 entries: &[
827 BindGroupEntry {
828 binding: 0,
829 resource: bucket_sizes_gpu.as_entire_binding(),
830 },
831 BindGroupEntry {
832 binding: 1,
833 resource: buckets_gpu.as_entire_binding(),
834 },
835 ],
836 });
837
838 (bind_group, compute_pipeline)
839}
840
841fn bind_group_and_pipeline_find_matches_and_compute_f2(
842 device: &wgpu::Device,
843 module: &ShaderModule,
844 parent_buckets_gpu: &Buffer,
845 bucket_sizes_gpu: &Buffer,
846 buckets_gpu: &Buffer,
847 positions_gpu: &Buffer,
848 metadatas_gpu: &Buffer,
849) -> (BindGroup, ComputePipeline) {
850 let bind_group_layout = device.create_bind_group_layout(&BindGroupLayoutDescriptor {
851 label: Some("find_matches_and_compute_f2"),
852 entries: &[
853 BindGroupLayoutEntry {
854 binding: 0,
855 count: None,
856 visibility: ShaderStages::COMPUTE,
857 ty: BindingType::Buffer {
858 has_dynamic_offset: false,
859 min_binding_size: None,
860 ty: BufferBindingType::Storage { read_only: true },
861 },
862 },
863 BindGroupLayoutEntry {
864 binding: 1,
865 count: None,
866 visibility: ShaderStages::COMPUTE,
867 ty: BindingType::Buffer {
868 has_dynamic_offset: false,
869 min_binding_size: None,
870 ty: BufferBindingType::Storage { read_only: false },
871 },
872 },
873 BindGroupLayoutEntry {
874 binding: 2,
875 count: None,
876 visibility: ShaderStages::COMPUTE,
877 ty: BindingType::Buffer {
878 has_dynamic_offset: false,
879 min_binding_size: None,
880 ty: BufferBindingType::Storage { read_only: false },
881 },
882 },
883 BindGroupLayoutEntry {
884 binding: 3,
885 count: None,
886 visibility: ShaderStages::COMPUTE,
887 ty: BindingType::Buffer {
888 has_dynamic_offset: false,
889 min_binding_size: None,
890 ty: BufferBindingType::Storage { read_only: false },
891 },
892 },
893 BindGroupLayoutEntry {
894 binding: 4,
895 count: None,
896 visibility: ShaderStages::COMPUTE,
897 ty: BindingType::Buffer {
898 has_dynamic_offset: false,
899 min_binding_size: None,
900 ty: BufferBindingType::Storage { read_only: false },
901 },
902 },
903 ],
904 });
905
906 let pipeline_layout = device.create_pipeline_layout(&PipelineLayoutDescriptor {
907 label: Some("find_matches_and_compute_f2"),
908 bind_group_layouts: &[Some(&bind_group_layout)],
909 immediate_size: 0,
910 });
911
912 let compute_pipeline = device.create_compute_pipeline(&ComputePipelineDescriptor {
913 compilation_options: PipelineCompilationOptions {
914 constants: &[],
915 zero_initialize_workgroup_memory: true,
916 },
917 cache: None,
918 label: Some("find_matches_and_compute_f2"),
919 layout: Some(&pipeline_layout),
920 module,
921 entry_point: Some("find_matches_and_compute_f2"),
922 });
923
924 let bind_group = device.create_bind_group(&BindGroupDescriptor {
925 label: Some("find_matches_and_compute_f2"),
926 layout: &bind_group_layout,
927 entries: &[
928 BindGroupEntry {
929 binding: 0,
930 resource: parent_buckets_gpu.as_entire_binding(),
931 },
932 BindGroupEntry {
933 binding: 1,
934 resource: bucket_sizes_gpu.as_entire_binding(),
935 },
936 BindGroupEntry {
937 binding: 2,
938 resource: buckets_gpu.as_entire_binding(),
939 },
940 BindGroupEntry {
941 binding: 3,
942 resource: positions_gpu.as_entire_binding(),
943 },
944 BindGroupEntry {
945 binding: 4,
946 resource: metadatas_gpu.as_entire_binding(),
947 },
948 ],
949 });
950
951 (bind_group, compute_pipeline)
952}
953
954#[expect(
955 clippy::too_many_arguments,
956 reason = "Both I/O and Vulkan stuff together take a lot of arguments"
957)]
958fn bind_group_and_pipeline_find_matches_and_compute_fn<const TABLE_NUMBER: u8>(
959 device: &wgpu::Device,
960 module: &ShaderModule,
961 parent_buckets_gpu: &Buffer,
962 parent_metadatas_gpu: &Buffer,
963 bucket_sizes_gpu: &Buffer,
964 buckets_gpu: &Buffer,
965 positions_gpu: &Buffer,
966 metadatas_gpu: &Buffer,
967) -> (BindGroup, ComputePipeline) {
968 let label = format!("find_matches_and_compute_f{TABLE_NUMBER}");
969 let bind_group_layout = device.create_bind_group_layout(&BindGroupLayoutDescriptor {
970 label: Some(&label),
971 entries: &[
972 BindGroupLayoutEntry {
973 binding: 0,
974 count: None,
975 visibility: ShaderStages::COMPUTE,
976 ty: BindingType::Buffer {
977 has_dynamic_offset: false,
978 min_binding_size: None,
979 ty: BufferBindingType::Storage { read_only: true },
980 },
981 },
982 BindGroupLayoutEntry {
983 binding: 1,
984 count: None,
985 visibility: ShaderStages::COMPUTE,
986 ty: BindingType::Buffer {
987 has_dynamic_offset: false,
988 min_binding_size: None,
989 ty: BufferBindingType::Storage { read_only: true },
990 },
991 },
992 BindGroupLayoutEntry {
993 binding: 2,
994 count: None,
995 visibility: ShaderStages::COMPUTE,
996 ty: BindingType::Buffer {
997 has_dynamic_offset: false,
998 min_binding_size: None,
999 ty: BufferBindingType::Storage { read_only: false },
1000 },
1001 },
1002 BindGroupLayoutEntry {
1003 binding: 3,
1004 count: None,
1005 visibility: ShaderStages::COMPUTE,
1006 ty: BindingType::Buffer {
1007 has_dynamic_offset: false,
1008 min_binding_size: None,
1009 ty: BufferBindingType::Storage { read_only: false },
1010 },
1011 },
1012 BindGroupLayoutEntry {
1013 binding: 4,
1014 count: None,
1015 visibility: ShaderStages::COMPUTE,
1016 ty: BindingType::Buffer {
1017 has_dynamic_offset: false,
1018 min_binding_size: None,
1019 ty: BufferBindingType::Storage { read_only: false },
1020 },
1021 },
1022 BindGroupLayoutEntry {
1023 binding: 5,
1024 count: None,
1025 visibility: ShaderStages::COMPUTE,
1026 ty: BindingType::Buffer {
1027 has_dynamic_offset: false,
1028 min_binding_size: None,
1029 ty: BufferBindingType::Storage { read_only: false },
1030 },
1031 },
1032 ],
1033 });
1034
1035 let pipeline_layout = device.create_pipeline_layout(&PipelineLayoutDescriptor {
1036 label: Some(&label),
1037 bind_group_layouts: &[Some(&bind_group_layout)],
1038 immediate_size: 0,
1039 });
1040
1041 let compute_pipeline = device.create_compute_pipeline(&ComputePipelineDescriptor {
1042 compilation_options: PipelineCompilationOptions {
1043 constants: &[],
1044 zero_initialize_workgroup_memory: true,
1045 },
1046 cache: None,
1047 label: Some(&label),
1048 layout: Some(&pipeline_layout),
1049 module,
1050 entry_point: Some(&format!("find_matches_and_compute_f{TABLE_NUMBER}")),
1051 });
1052
1053 let bind_group = device.create_bind_group(&BindGroupDescriptor {
1054 label: Some(&label),
1055 layout: &bind_group_layout,
1056 entries: &[
1057 BindGroupEntry {
1058 binding: 0,
1059 resource: parent_buckets_gpu.as_entire_binding(),
1060 },
1061 BindGroupEntry {
1062 binding: 1,
1063 resource: parent_metadatas_gpu.as_entire_binding(),
1064 },
1065 BindGroupEntry {
1066 binding: 2,
1067 resource: bucket_sizes_gpu.as_entire_binding(),
1068 },
1069 BindGroupEntry {
1070 binding: 3,
1071 resource: buckets_gpu.as_entire_binding(),
1072 },
1073 BindGroupEntry {
1074 binding: 4,
1075 resource: positions_gpu.as_entire_binding(),
1076 },
1077 BindGroupEntry {
1078 binding: 5,
1079 resource: metadatas_gpu.as_entire_binding(),
1080 },
1081 ],
1082 });
1083
1084 (bind_group, compute_pipeline)
1085}
1086
1087fn bind_group_and_pipeline_find_matches_and_compute_f7(
1088 device: &wgpu::Device,
1089 module: &ShaderModule,
1090 parent_buckets_gpu: &Buffer,
1091 parent_metadatas_gpu: &Buffer,
1092 table_6_proof_targets_sizes_gpu: &Buffer,
1093 table_6_proof_targets_gpu: &Buffer,
1094) -> (BindGroup, ComputePipeline) {
1095 let bind_group_layout = device.create_bind_group_layout(&BindGroupLayoutDescriptor {
1096 label: Some("find_matches_and_compute_f7"),
1097 entries: &[
1098 BindGroupLayoutEntry {
1099 binding: 0,
1100 count: None,
1101 visibility: ShaderStages::COMPUTE,
1102 ty: BindingType::Buffer {
1103 has_dynamic_offset: false,
1104 min_binding_size: None,
1105 ty: BufferBindingType::Storage { read_only: true },
1106 },
1107 },
1108 BindGroupLayoutEntry {
1109 binding: 1,
1110 count: None,
1111 visibility: ShaderStages::COMPUTE,
1112 ty: BindingType::Buffer {
1113 has_dynamic_offset: false,
1114 min_binding_size: None,
1115 ty: BufferBindingType::Storage { read_only: true },
1116 },
1117 },
1118 BindGroupLayoutEntry {
1119 binding: 2,
1120 count: None,
1121 visibility: ShaderStages::COMPUTE,
1122 ty: BindingType::Buffer {
1123 has_dynamic_offset: false,
1124 min_binding_size: None,
1125 ty: BufferBindingType::Storage { read_only: false },
1126 },
1127 },
1128 BindGroupLayoutEntry {
1129 binding: 3,
1130 count: None,
1131 visibility: ShaderStages::COMPUTE,
1132 ty: BindingType::Buffer {
1133 has_dynamic_offset: false,
1134 min_binding_size: None,
1135 ty: BufferBindingType::Storage { read_only: false },
1136 },
1137 },
1138 ],
1139 });
1140
1141 let pipeline_layout = device.create_pipeline_layout(&PipelineLayoutDescriptor {
1142 label: Some("find_matches_and_compute_f7"),
1143 bind_group_layouts: &[Some(&bind_group_layout)],
1144 immediate_size: 0,
1145 });
1146
1147 let compute_pipeline = device.create_compute_pipeline(&ComputePipelineDescriptor {
1148 compilation_options: PipelineCompilationOptions {
1149 constants: &[],
1150 zero_initialize_workgroup_memory: true,
1151 },
1152 cache: None,
1153 label: Some("find_matches_and_compute_f7"),
1154 layout: Some(&pipeline_layout),
1155 module,
1156 entry_point: Some("find_matches_and_compute_f7"),
1157 });
1158
1159 let bind_group = device.create_bind_group(&BindGroupDescriptor {
1160 label: Some("find_matches_and_compute_f7"),
1161 layout: &bind_group_layout,
1162 entries: &[
1163 BindGroupEntry {
1164 binding: 0,
1165 resource: parent_buckets_gpu.as_entire_binding(),
1166 },
1167 BindGroupEntry {
1168 binding: 1,
1169 resource: parent_metadatas_gpu.as_entire_binding(),
1170 },
1171 BindGroupEntry {
1172 binding: 2,
1173 resource: table_6_proof_targets_sizes_gpu.as_entire_binding(),
1174 },
1175 BindGroupEntry {
1176 binding: 3,
1177 resource: table_6_proof_targets_gpu.as_entire_binding(),
1178 },
1179 ],
1180 });
1181
1182 (bind_group, compute_pipeline)
1183}
1184
1185#[expect(
1186 clippy::too_many_arguments,
1187 reason = "Both I/O and Vulkan stuff together take a lot of arguments"
1188)]
1189fn bind_group_and_pipeline_find_proofs(
1190 device: &wgpu::Device,
1191 module: &ShaderModule,
1192 table_2_positions_gpu: &Buffer,
1193 table_3_positions_gpu: &Buffer,
1194 table_4_positions_gpu: &Buffer,
1195 table_5_positions_gpu: &Buffer,
1196 table_6_positions_gpu: &Buffer,
1197 bucket_sizes_gpu: &Buffer,
1198 buckets_gpu: &Buffer,
1199 proofs_gpu: &Buffer,
1200) -> (BindGroup, ComputePipeline) {
1201 let bind_group_layout = device.create_bind_group_layout(&BindGroupLayoutDescriptor {
1202 label: Some("find_proofs"),
1203 entries: &[
1204 BindGroupLayoutEntry {
1205 binding: 0,
1206 count: None,
1207 visibility: ShaderStages::COMPUTE,
1208 ty: BindingType::Buffer {
1209 has_dynamic_offset: false,
1210 min_binding_size: None,
1211 ty: BufferBindingType::Storage { read_only: true },
1212 },
1213 },
1214 BindGroupLayoutEntry {
1215 binding: 1,
1216 count: None,
1217 visibility: ShaderStages::COMPUTE,
1218 ty: BindingType::Buffer {
1219 has_dynamic_offset: false,
1220 min_binding_size: None,
1221 ty: BufferBindingType::Storage { read_only: true },
1222 },
1223 },
1224 BindGroupLayoutEntry {
1225 binding: 2,
1226 count: None,
1227 visibility: ShaderStages::COMPUTE,
1228 ty: BindingType::Buffer {
1229 has_dynamic_offset: false,
1230 min_binding_size: None,
1231 ty: BufferBindingType::Storage { read_only: true },
1232 },
1233 },
1234 BindGroupLayoutEntry {
1235 binding: 3,
1236 count: None,
1237 visibility: ShaderStages::COMPUTE,
1238 ty: BindingType::Buffer {
1239 has_dynamic_offset: false,
1240 min_binding_size: None,
1241 ty: BufferBindingType::Storage { read_only: true },
1242 },
1243 },
1244 BindGroupLayoutEntry {
1245 binding: 4,
1246 count: None,
1247 visibility: ShaderStages::COMPUTE,
1248 ty: BindingType::Buffer {
1249 has_dynamic_offset: false,
1250 min_binding_size: None,
1251 ty: BufferBindingType::Storage { read_only: true },
1252 },
1253 },
1254 BindGroupLayoutEntry {
1255 binding: 5,
1256 count: None,
1257 visibility: ShaderStages::COMPUTE,
1258 ty: BindingType::Buffer {
1259 has_dynamic_offset: false,
1260 min_binding_size: None,
1261 ty: BufferBindingType::Storage { read_only: false },
1262 },
1263 },
1264 BindGroupLayoutEntry {
1265 binding: 6,
1266 count: None,
1267 visibility: ShaderStages::COMPUTE,
1268 ty: BindingType::Buffer {
1269 has_dynamic_offset: false,
1270 min_binding_size: None,
1271 ty: BufferBindingType::Storage { read_only: true },
1272 },
1273 },
1274 BindGroupLayoutEntry {
1275 binding: 7,
1276 count: None,
1277 visibility: ShaderStages::COMPUTE,
1278 ty: BindingType::Buffer {
1279 has_dynamic_offset: false,
1280 min_binding_size: None,
1281 ty: BufferBindingType::Storage { read_only: false },
1282 },
1283 },
1284 ],
1285 });
1286
1287 let pipeline_layout = device.create_pipeline_layout(&PipelineLayoutDescriptor {
1288 label: Some("find_proofs"),
1289 bind_group_layouts: &[Some(&bind_group_layout)],
1290 immediate_size: 0,
1291 });
1292
1293 let compute_pipeline = device.create_compute_pipeline(&ComputePipelineDescriptor {
1294 compilation_options: PipelineCompilationOptions {
1295 constants: &[],
1296 zero_initialize_workgroup_memory: false,
1297 },
1298 cache: None,
1299 label: Some("find_proofs"),
1300 layout: Some(&pipeline_layout),
1301 module,
1302 entry_point: Some("find_proofs"),
1303 });
1304
1305 let bind_group = device.create_bind_group(&BindGroupDescriptor {
1306 label: Some("find_proofs"),
1307 layout: &bind_group_layout,
1308 entries: &[
1309 BindGroupEntry {
1310 binding: 0,
1311 resource: table_2_positions_gpu.as_entire_binding(),
1312 },
1313 BindGroupEntry {
1314 binding: 1,
1315 resource: table_3_positions_gpu.as_entire_binding(),
1316 },
1317 BindGroupEntry {
1318 binding: 2,
1319 resource: table_4_positions_gpu.as_entire_binding(),
1320 },
1321 BindGroupEntry {
1322 binding: 3,
1323 resource: table_5_positions_gpu.as_entire_binding(),
1324 },
1325 BindGroupEntry {
1326 binding: 4,
1327 resource: table_6_positions_gpu.as_entire_binding(),
1328 },
1329 BindGroupEntry {
1330 binding: 5,
1331 resource: bucket_sizes_gpu.as_entire_binding(),
1332 },
1333 BindGroupEntry {
1334 binding: 6,
1335 resource: buckets_gpu.as_entire_binding(),
1336 },
1337 BindGroupEntry {
1338 binding: 7,
1339 resource: proofs_gpu.as_entire_binding(),
1340 },
1341 ],
1342 });
1343
1344 (bind_group, compute_pipeline)
1345}