subspace_farmer_components/
file_ext.rs

1//! File extension trait
2
3use std::fs::{File, OpenOptions};
4use std::io::Result;
5
6/// Extension convenience trait that allows setting some file opening options in cross-platform way
7pub trait OpenOptionsExt {
8    /// Advise OS/file system that file will use random access and read-ahead behavior is
9    /// undesirable, only has impact on Windows, for other operating systems see [`FileExt`]
10    fn advise_random_access(&mut self) -> &mut Self;
11
12    /// Advise OS/file system that file will use sequential access and read-ahead behavior is
13    /// desirable, only has impact on Windows, for other operating systems see [`FileExt`]
14    fn advise_sequential_access(&mut self) -> &mut Self;
15
16    /// Use Direct I/O on Linux and disable buffering on Windows.
17    ///
18    /// NOTE: There are major alignment requirements described here:
19    /// <https://learn.microsoft.com/en-us/windows/win32/fileio/file-buffering#alignment-and-file-access-requirements>
20    /// <https://man7.org/linux/man-pages/man2/open.2.html>
21    fn use_direct_io(&mut self) -> &mut Self;
22}
23
24impl OpenOptionsExt for OpenOptions {
25    #[cfg(not(windows))]
26    fn advise_random_access(&mut self) -> &mut Self {
27        // Not supported
28        self
29    }
30
31    #[cfg(windows)]
32    fn advise_random_access(&mut self) -> &mut Self {
33        use std::os::windows::fs::OpenOptionsExt;
34        // `FILE_FLAG_WRITE_THROUGH` below is a bit of a hack, especially in `advise_random_access`,
35        // but it helps with memory usage and feels like should be default. Since `.custom_flags()`
36        // overrides previous value, we need to set bitwise OR of two flags rather that two flags
37        // separately.
38        self.custom_flags(
39            winapi::um::winbase::FILE_FLAG_RANDOM_ACCESS
40                | winapi::um::winbase::FILE_FLAG_WRITE_THROUGH,
41        )
42    }
43
44    #[cfg(not(windows))]
45    fn advise_sequential_access(&mut self) -> &mut Self {
46        // Not supported
47        self
48    }
49
50    #[cfg(windows)]
51    fn advise_sequential_access(&mut self) -> &mut Self {
52        use std::os::windows::fs::OpenOptionsExt;
53        self.custom_flags(winapi::um::winbase::FILE_FLAG_SEQUENTIAL_SCAN)
54    }
55
56    #[cfg(windows)]
57    fn use_direct_io(&mut self) -> &mut Self {
58        use std::os::windows::fs::OpenOptionsExt;
59        self.custom_flags(
60            winapi::um::winbase::FILE_FLAG_WRITE_THROUGH
61                | winapi::um::winbase::FILE_FLAG_NO_BUFFERING,
62        )
63    }
64
65    #[cfg(target_os = "linux")]
66    fn use_direct_io(&mut self) -> &mut Self {
67        use std::os::unix::fs::OpenOptionsExt;
68        self.custom_flags(libc::O_DIRECT)
69    }
70
71    #[cfg(not(any(target_os = "linux", windows)))]
72    fn use_direct_io(&mut self) -> &mut Self {
73        // Not supported
74        self
75    }
76}
77
78/// Extension convenience trait that allows pre-allocating files, suggesting random access pattern
79/// and doing cross-platform exact reads/writes
80pub trait FileExt {
81    /// Get file size
82    fn size(&self) -> Result<u64>;
83
84    /// Make sure file has specified number of bytes allocated for it
85    fn preallocate(&self, len: u64) -> Result<()>;
86
87    /// Advise OS/file system that file will use random access and read-ahead behavior is
88    /// undesirable, on Windows this can only be set when file is opened, see [`OpenOptionsExt`]
89    fn advise_random_access(&self) -> Result<()>;
90
91    /// Advise OS/file system that file will use sequential access and read-ahead behavior is
92    /// desirable, on Windows this can only be set when file is opened, see [`OpenOptionsExt`]
93    fn advise_sequential_access(&self) -> Result<()>;
94
95    /// Disable cache on macOS
96    fn disable_cache(&self) -> Result<()>;
97
98    /// Read exact number of bytes at a specific offset
99    fn read_exact_at(&self, buf: &mut [u8], offset: u64) -> Result<()>;
100
101    /// Write all provided bytes at a specific offset
102    fn write_all_at(&self, buf: &[u8], offset: u64) -> Result<()>;
103}
104
105impl FileExt for File {
106    fn size(&self) -> Result<u64> {
107        Ok(self.metadata()?.len())
108    }
109
110    fn preallocate(&self, len: u64) -> Result<()> {
111        fs2::FileExt::allocate(self, len)
112    }
113
114    #[cfg(target_os = "linux")]
115    fn advise_random_access(&self) -> Result<()> {
116        use std::os::unix::io::AsRawFd;
117        let err = unsafe { libc::posix_fadvise(self.as_raw_fd(), 0, 0, libc::POSIX_FADV_RANDOM) };
118        if err != 0 {
119            Err(std::io::Error::from_raw_os_error(err))
120        } else {
121            Ok(())
122        }
123    }
124
125    #[cfg(target_os = "macos")]
126    fn advise_random_access(&self) -> Result<()> {
127        use std::os::unix::io::AsRawFd;
128        if unsafe { libc::fcntl(self.as_raw_fd(), libc::F_RDAHEAD, 0) } != 0 {
129            Err(std::io::Error::last_os_error())
130        } else {
131            Ok(())
132        }
133    }
134
135    #[cfg(windows)]
136    fn advise_random_access(&self) -> Result<()> {
137        // Not supported
138        Ok(())
139    }
140
141    #[cfg(target_os = "linux")]
142    fn advise_sequential_access(&self) -> Result<()> {
143        use std::os::unix::io::AsRawFd;
144        let err =
145            unsafe { libc::posix_fadvise(self.as_raw_fd(), 0, 0, libc::POSIX_FADV_SEQUENTIAL) };
146        if err != 0 {
147            Err(std::io::Error::from_raw_os_error(err))
148        } else {
149            Ok(())
150        }
151    }
152
153    #[cfg(target_os = "macos")]
154    fn advise_sequential_access(&self) -> Result<()> {
155        use std::os::unix::io::AsRawFd;
156        if unsafe { libc::fcntl(self.as_raw_fd(), libc::F_RDAHEAD, 1) } != 0 {
157            Err(std::io::Error::last_os_error())
158        } else {
159            Ok(())
160        }
161    }
162
163    #[cfg(windows)]
164    fn advise_sequential_access(&self) -> Result<()> {
165        // Not supported
166        Ok(())
167    }
168
169    #[cfg(not(target_os = "macos"))]
170    fn disable_cache(&self) -> Result<()> {
171        // Not supported
172        Ok(())
173    }
174
175    #[cfg(target_os = "macos")]
176    fn disable_cache(&self) -> Result<()> {
177        use std::os::unix::io::AsRawFd;
178        if unsafe { libc::fcntl(self.as_raw_fd(), libc::F_NOCACHE, 1) } != 0 {
179            Err(std::io::Error::last_os_error())
180        } else {
181            Ok(())
182        }
183    }
184
185    #[cfg(unix)]
186    fn read_exact_at(&self, buf: &mut [u8], offset: u64) -> Result<()> {
187        std::os::unix::fs::FileExt::read_exact_at(self, buf, offset)
188    }
189
190    #[cfg(unix)]
191    fn write_all_at(&self, buf: &[u8], offset: u64) -> Result<()> {
192        std::os::unix::fs::FileExt::write_all_at(self, buf, offset)
193    }
194
195    #[cfg(windows)]
196    fn read_exact_at(&self, mut buf: &mut [u8], mut offset: u64) -> Result<()> {
197        while !buf.is_empty() {
198            match std::os::windows::fs::FileExt::seek_read(self, buf, offset) {
199                Ok(0) => {
200                    break;
201                }
202                Ok(n) => {
203                    buf = &mut buf[n..];
204                    offset += n as u64;
205                }
206                Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => {
207                    // Try again
208                }
209                Err(e) => {
210                    return Err(e);
211                }
212            }
213        }
214
215        if !buf.is_empty() {
216            Err(std::io::Error::new(
217                std::io::ErrorKind::UnexpectedEof,
218                "failed to fill whole buffer",
219            ))
220        } else {
221            Ok(())
222        }
223    }
224
225    #[cfg(windows)]
226    fn write_all_at(&self, mut buf: &[u8], mut offset: u64) -> Result<()> {
227        while !buf.is_empty() {
228            match std::os::windows::fs::FileExt::seek_write(self, buf, offset) {
229                Ok(0) => {
230                    return Err(std::io::Error::new(
231                        std::io::ErrorKind::WriteZero,
232                        "failed to write whole buffer",
233                    ));
234                }
235                Ok(n) => {
236                    buf = &buf[n..];
237                    offset += n as u64;
238                }
239                Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => {
240                    // Try again
241                }
242                Err(e) => {
243                    return Err(e);
244                }
245            }
246        }
247
248        Ok(())
249    }
250}
subspace_farmer_components/file_ext.rs

subspace_farmer_components/
file_ext.rs
