Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions .github/workflows/r.yml
Original file line number Diff line number Diff line change
Expand Up @@ -228,11 +228,10 @@ jobs:
r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-
- uses: r-lib/actions/setup-r@v2
with:
# Note: RTools must be 40 here because RTools40 + ucrt is how we build the Arrow C++
# static library. The R is not used here but R 4.1 was the last R to use
# Rtools40.
# Note: we need Rtools40 here because Rtools40 + ucrt is how we build the Arrow C++
# static library. R itself is not used, but setup-r picks the Rtools version from
# r-version, and R 4.1 was the last R to use Rtools40, so this must stay at 4.1.
r-version: "4.1"
rtools-version: ${{ matrix.config.rtools }}
Ncpus: 2
- name: Build Arrow C++
shell: bash
Expand Down
3 changes: 0 additions & 3 deletions ci/docker/linux-r.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,6 @@ ENV R_PRUNE_DEPS=${r_prune_deps}
ARG r_custom_ccache=false
ENV R_CUSTOM_CCACHE=${r_custom_ccache}

ARG r_update_clang=false
ENV R_UPDATE_CLANG=${r_update_clang}

ARG tz="UTC"
ENV TZ=${tz}

Expand Down
18 changes: 0 additions & 18 deletions ci/scripts/r_docker_configure.sh
Original file line number Diff line number Diff line change
Expand Up @@ -90,23 +90,5 @@ if [ "$PACKAGE_MANAGER" = "apt-get" ] && apt-cache show tzdata-legacy > /dev/nul
apt-get install -y tzdata-legacy
fi


# Update clang version to latest available.
# The rhub/ubuntu-clang image ships clang-15 but CRAN's debian-clang now uses
# clang 22. Install clang-22 from LLVM's apt repos to match.
if [ "$R_UPDATE_CLANG" = true ]; then
apt-get update -y --allow-releaseinfo-change
apt-get install -y gnupg
curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor -o /etc/apt/trusted.gpg.d/llvm.gpg
echo "deb https://apt.llvm.org/jammy/ llvm-toolchain-jammy-22 main" > /etc/apt/sources.list.d/llvm22.list
apt-get update -y --allow-releaseinfo-change
apt-get install -y clang-22 lld-22
# The rhub image hardcodes clang-15/clang++-15 in R's Makeconf, so installing
# clang-22 alone isn't enough: point R at the new compilers.
sed -i 's/clang++-15/clang++-22/g; s/clang-15/clang-22/g' "$(R RHOME)/etc/Makeconf"
echo "R is now using CC=$(R CMD config CC) CXX=$(R CMD config CXX)"
clang-22 --version
fi

# Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786
Rscript -e 'x <- file.path(R.home("doc"), "html"); if (!file.exists(x)) {dir.create(x, recursive=TRUE); file.copy(system.file("html/R.css", package="stats"), x)}'
48 changes: 47 additions & 1 deletion ci/scripts/r_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,22 @@ fi
export TEXMFCONFIG=/tmp/texmf-config
export TEXMFVAR=/tmp/texmf-var

if [ "${ARROW_R_TEST_OFFLINE:-false}" = "true" ]; then
# Simulate a machine without network access (e.g. a CRAN builder whose
# downloads fail) by disabling DNS resolution, like ubuntu-cpp-bundled-offline.
# We can't use mv because we can't remove /etc/resolv.conf in Docker
# container.
cp /etc/resolv.conf{,.bak}
echo > /etc/resolv.conf
# sccache can't reach its remote cache without network access
unset SCCACHE_BUCKET SCCACHE_REGION SCCACHE_S3_KEY_PREFIX
# Make sure we actually are offline so this check can't silently pass
if ${R_BIN} -s -e 'readLines(url("https://github.com"), n = 1L)' > /dev/null 2>&1; then
echo "Network is still reachable, but ARROW_R_TEST_OFFLINE=true"
exit 1
fi
fi

# Make sure we aren't writing to the home dir (CRAN _hates_ this but there is no official check)
BEFORE=$(ls -alh ~/)

Expand Down Expand Up @@ -125,13 +141,43 @@ SCRIPT="as_cran <- !identical(tolower(Sys.getenv('NOT_CRAN')), 'true')
args <- c(args, paste0('--install-args=\"', install_args, '\"'))
}

# Without network access, some R CMD check steps (e.g. filtering undeclared
# '::' imports in tests against CRAN, resolving Rd xrefs) produce spurious
# warnings, so only fail on errors when offline.
offline <- identical(tolower(Sys.getenv('ARROW_R_TEST_OFFLINE')), 'true')
error_on <- if (offline) 'error' else 'warning'

message('Running rcmdcheck with:\n')
print(build_args)
print(args)

rcmdcheck::rcmdcheck(build_args = build_args, args = args, error_on = 'warning', check_dir = 'check', timeout = 3600)"
rcmdcheck::rcmdcheck(build_args = build_args, args = args, error_on = error_on, check_dir = 'check', timeout = 3600)"
echo "$SCRIPT" | "${R_BIN}" --no-save

if [ "${ARROW_R_TEST_OFFLINE:-false}" = "true" ]; then
cp /etc/resolv.conf{.bak,}
fi

if [ -n "${ARROW_R_EXPECTED_FEATURES:-}" ]; then
# The tests skip what the build doesn't support, so make sure the features
# we expect (e.g. from the vendored thirdparty sources) were actually built.
# ARROW_R_EXPECTED_FEATURES is a comma-separated list of
# names(arrow::arrow_info()$capabilities) and memory pool backends
# (e.g. mimalloc).
${R_BIN} -s -e "
library(arrow, lib.loc = 'check/arrow.Rcheck')
info <- arrow_info()
print(info\$capabilities)
print(info\$memory_pool\$available_backends)
available <- c(names(which(info\$capabilities)), info\$memory_pool\$available_backends)
expected <- strsplit(Sys.getenv('ARROW_R_EXPECTED_FEATURES'), ',')[[1]]
missing <- setdiff(expected, available)
if (length(missing) > 0) {
stop('Expected features are not available: ', paste(missing, collapse = ', '))
}
"
fi

AFTER=$(ls -alh ~/)
if [ "$NOT_CRAN" != "true" ] && [ "$BEFORE" != "$AFTER" ]; then
# Ignore ~/.TinyTex/ and ~/R/ because it has many files.
Expand Down
1 change: 0 additions & 1 deletion compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1686,7 +1686,6 @@ services:
tz: ${TZ}
r_prune_deps: ${R_PRUNE_DEPS}
r_custom_ccache: ${R_CUSTOM_CCACHE}
r_update_clang: ${R_UPDATE_CLANG}
shm_size: *shm-size
environment:
<<: [*common, *sccache]
Expand Down
6 changes: 6 additions & 0 deletions cpp/cmake_modules/ThirdpartyToolchain.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -469,6 +469,12 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
continue()
endif()

# Allow overriding checksums by environment variables, e.g. for a source
# archive given by ARROW_$LIBRARY_URL that differs from the upstream one
if(_VARIABLE_NAME MATCHES "_CHECKSUM$" AND DEFINED ENV{${_VARIABLE_NAME}})
set(_VARIABLE_VALUE "$ENV{${_VARIABLE_NAME}}")
endif()

# For debugging
message(STATUS "${_VARIABLE_NAME}: ${_VARIABLE_VALUE}")

Expand Down
1 change: 1 addition & 0 deletions dev/release/rat_exclude_files.txt
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ r/tests/testthat/_snaps/*.md
r/inst/include/cpp11.hpp
r/inst/include/cpp11/*.hpp
r/tools/nixlibs-allowlist.txt
r/inst/COPYRIGHTS
.gitattributes
ruby/red-arrow/.yardopts
ruby/red-arrow-format/lib/arrow-format/org/*
Expand Down
5 changes: 2 additions & 3 deletions dev/tasks/r/github.linux.cran.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,16 +35,15 @@ jobs:
config:
# See https://r-hub.github.io/containers/
- { r_image: "ubuntu-gcc12" } # ~ r-devel-linux-x86_64-debian-gcc
- { r_image: "ubuntu-clang", r_update_clang: true } # ~ r-devel-linux-x86_64-debian-clang
- { r_image: "ubuntu-clang" } # ~ r-devel-linux-x86_64-debian-clang
- { r_image: "ubuntu-next" } # ~ r-patched-linux-x86_64
- { r_image: "ubuntu-release" } # ~ r-release-linux-x86_64
- { r_image: "clang22", skip_vignettes: true } # ~ r-devel-linux-x86_64-fedora-clang
- { r_image: "clang23", skip_vignettes: true } # ~ r-devel-linux-x86_64-fedora-clang
env:
R_ORG: "rhub"
R_IMAGE: {{ '${{ matrix.config.r_image }}' }}
R_TAG: "latest"
ARROW_R_DEV: "FALSE"
R_UPDATE_CLANG: {{ '${{ matrix.config.r_update_clang }}' }}
steps:
{{ macros.github_checkout_arrow()|indent }}
{{ macros.github_install_archery()|indent }}
Expand Down
5 changes: 3 additions & 2 deletions dev/tasks/r/github.packages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -183,8 +183,9 @@ jobs:
{{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }}
- uses: r-lib/actions/setup-r@v2
with:
rtools-version: 40
r-version: "4.0"
# R 4.1 was the last R to use Rtools40, which is how we build the
# Arrow C++ static library. setup-r installs Rtools40 for it.
r-version: "4.1"
Ncpus: 2
- name: Install sccache
shell: bash
Expand Down
24 changes: 24 additions & 0 deletions dev/tasks/tasks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -632,6 +632,30 @@ tasks:
ci: github
template: r/github.linux.offline.build.yml

# Simulate a CRAN builder without network access: the build should detect
# that it is offline and build libarrow without downloading anything, using
# the thirdparty sources vendored in the package (tools/vendored) to still
# include Parquet, Dataset, JSON, the compression codecs, etc.
# ARROW_SOURCE_HOME='' makes it use the C++ source copied into tools/cpp
# (like the CRAN package does), whose versions.txt has the checksums of the
# trimmed tarballs. bz2 is off by default on Linux, so it is turned on to make
# sure every vendored tarball is built. Brotli and Zstandard aren't vendored.
test-r-offline-vendored:
ci: github
template: docker-tests/github.linux.yml
params:
flags: >-
-e ARROW_R_TEST_OFFLINE=true
-e NOT_CRAN=false
-e SKIP_VIGNETTES=true
-e ARROW_SOURCE_HOME=''
-e ARROW_WITH_BZ2=ON
-e ARROW_R_EXPECTED_FEATURES=dataset,parquet,json,mimalloc,re2,utf8proc,snappy,gzip,lz4,bz2
image: r
# A full libarrow build with 2 cores (NOT_CRAN=false) and no sccache
# (it's offline)
timeout: 180

# This one has -flto=auto
test-r-ubuntu-22.04:
ci: github
Expand Down
4 changes: 4 additions & 0 deletions docs/source/developers/cpp/building.rst
Original file line number Diff line number Diff line change
Expand Up @@ -581,6 +581,10 @@ internet.
To enable offline builds, you can download the source artifacts yourself and
use environment variables of the form ``ARROW_$LIBRARY_URL`` to direct the
build system to read from a local file rather than accessing the internet.
Each archive is checked against the SHA256 checksum in
``thirdparty/versions.txt``; if you use an archive that differs from the
upstream one, you can set the matching ``ARROW_$LIBRARY_BUILD_SHA256_CHECKSUM``
environment variable to its checksum.

To make this easier for you, we have prepared a script
``thirdparty/download_dependencies.sh`` which will download the correct version
Expand Down
4 changes: 3 additions & 1 deletion r/.Rbuildignore
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ LICENSE.md
^data-raw$
lint.sh
Dockerfile
.*\.tar\.gz
^[^/]*\.tar\.gz$
^windows
^libarrow
^revdep
Expand All @@ -32,3 +32,5 @@ STYLE.md
^inst/__pycache__$
^bootstrap.R$
air.toml
^tools/vendored-downloads$
^tools/trim$
5 changes: 5 additions & 0 deletions r/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,8 @@ extra-tests/files
/tools/LICENSE.txt
/tools/NOTICE.txt
/tools/versions.txt
# Source tarballs of required dependencies that we ship for offline builds.
# They're downloaded by `make sync-cpp` (see tools/vendor_dependencies.sh).
/tools/vendored/
# Cache of the upstream (untrimmed) tarballs used by tools/vendor_dependencies.sh
/tools/vendored-downloads/
3 changes: 2 additions & 1 deletion r/DESCRIPTION
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
Package: arrow
Title: Integration to 'Apache' 'Arrow'
Version: 26.0.1
Version: 26.0.1.1
Authors@R: c(
person("Neal", "Richardson", email = "neal.p.richardson@gmail.com", role = c("aut")),
person("Ian", "Cook", email = "ianmcook@gmail.com", role = c("aut")),
Expand All @@ -24,6 +24,7 @@ Description: 'Apache' 'Arrow' <https://arrow.apache.org/> is a cross-language
package provides an interface to the 'Arrow C++' library.
Depends: R (>= 4.2)
License: Apache License (>= 2.0)
Copyright: See file inst/COPYRIGHTS
URL: https://github.com/apache/arrow/, https://arrow.apache.org/docs/r/
BugReports: https://github.com/apache/arrow/issues
Encoding: UTF-8
Expand Down
1 change: 1 addition & 0 deletions r/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ sync-cpp:
cp -p ../NOTICE.txt tools/
cp -p ../LICENSE.txt tools/
sed -i"" -e "s/\.env/dotenv/g" tools/cpp/CMakeLists.txt
bash tools/vendor_dependencies.sh

build:
$(MAKE) clean
Expand Down
6 changes: 6 additions & 0 deletions r/NEWS.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@
under the License.
-->

# arrow 26.0.1.1

- Third party dependency source is now included in the package to produce a
more fully-featured build of arrow even in circumstances where the
installation is offline.

# arrow 26.0.1

## Breaking changes
Expand Down
2 changes: 1 addition & 1 deletion r/R/dplyr-funcs-doc.R
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@
#' Functions can be called either as `pkg::fun()` or just `fun()`, i.e. both
#' `str_sub()` and `stringr::str_sub()` work.
#'
#' In addition to these functions, you can call any of Arrow's 283 compute
#' In addition to these functions, you can call any of Arrow's 287 compute
#' functions directly. Arrow has many functions that don't map to an existing R
#' function. In other cases where there is an R function mapping, you can still
#' call the Arrow function directly if you don't want the adaptations that the R
Expand Down
9 changes: 0 additions & 9 deletions r/README.md
Original file line number Diff line number Diff line change
@@ -1,14 +1,5 @@
# arrow <img src="https://arrow.apache.org/img/arrow-logo_hex_black-txt_white-bg.png" align="right" alt="" width="120" />

<!-- badges: start -->

[![cran](https://www.r-pkg.org/badges/version-last-release/arrow)](https://cran.r-project.org/package=arrow)
[![CI](https://github.com/apache/arrow/actions/workflows/r.yml/badge.svg?branch=main&event=push)](https://github.com/apache/arrow/actions/workflows/r.yml?query=branch%3Amain+event%3Apush)
[![R-universe status badge](https://apache.r-universe.dev/badges/arrow)](https://apache.r-universe.dev)
[![conda-forge](https://img.shields.io/conda/vn/conda-forge/r-arrow.svg)](https://anaconda.org/conda-forge/r-arrow)

<!-- badges: end -->

## Overview

The R `{arrow}` package provides access to many of the features of the [Apache Arrow C++ library](https://arrow.apache.org/docs/cpp/index.html) for R users. The goal of arrow is to provide an Arrow C++ backend to `{dplyr}`, and access to the Arrow C++ library through familiar base R and tidyverse functions, or `{R6}` classes. The dedicated R package website is located [here](https://arrow.apache.org/docs/r/index.html).
Expand Down
71 changes: 71 additions & 0 deletions r/inst/COPYRIGHTS
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
The arrow R package is Copyright The Apache Software Foundation and is licensed
under the Apache License, Version 2.0. The Arrow C++ library source copied into
tools/cpp includes code from other projects; see tools/LICENSE.txt and
tools/NOTICE.txt in the source package.

To build the Arrow C++ library without network access, the source package also
includes (trimmed) source tarballs of these thirdparty dependencies in
tools/vendored. They are only used when the dependency isn't found on the
system, and are statically linked into the package when used. Each tarball
contains the dependency's full license text.

Boost (the subset of libraries used by Apache Thrift; Boost.Locale and the
Boost libraries it needs are compiled and statically linked, the rest are
header-only)
Copyright the Boost authors
Boost Software License 1.0

bzip2
Copyright (C) 1996-2019 Julian R Seward
BSD-style license (see LICENSE in the tarball)

LZ4 (lib/ only)
Copyright (c) 2011-2020, Yann Collet
BSD 2-Clause License

mimalloc
Copyright (c) 2018-2025 Microsoft Corporation, Daan Leijen
MIT License

RapidJSON
Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip
MIT License (with code under the BSD and JSON licenses, see license.txt in
the tarball)

RE2
Copyright (c) 2009 The RE2 Authors
BSD 3-Clause License

simdjson
Copyright 2018-2025 The simdjson authors
Apache License 2.0 or MIT License

Snappy
Copyright 2011, Google Inc.
BSD 3-Clause License

Apache Thrift
Copyright (C) 2006-2019, The Apache Software Foundation
Apache License 2.0

uriparser (library sources only)
Copyright (C) 2007, Weijia Song and Sebastian Pipping
BSD 3-Clause License

utf8proc
Copyright (c) 2014-2021 Steven G. Johnson, Jiahao Chen, Tony Kelman,
Jonas Fonseca, and other contributors
Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany
MIT License, with Unicode data under the Unicode Data Files and Software
License

xsimd
Copyright (c) 2016, Johan Mabille, Sylvain Corlay, Wolf Vollprecht and
Martin Renou
Copyright (c) 2016, QuantStack
Copyright (c) 2018, Serge Guelton
BSD 3-Clause License

zlib
Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler
zlib License
2 changes: 1 addition & 1 deletion r/man/acero.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
239d1507cdce2753690f1cd86e2dabae96d8168bbe98b0ec8154dee7afc81400ddcd0903cce59adc75f9f12407e516a9a05ee24d3639d6b0abc2bcdc4879555b r-libarrow-darwin-arm64-26.0.1.zip
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
20813caea1f4b0a4a2ddca64c07071c18db373d2f7fa2119585f915c7b9ebb211c13167889f06f38843cad6f6fdfb30ff2bcf4ba51ee15697625a2762f49dc7a r-libarrow-darwin-x86_64-26.0.1.zip
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
d60746b29631e34ec77602e96f13e80f5576168ba90a4fa76e0c0f701ec439ee7ff518b3110e5312a353465e966c9ed50af26973f13a5af4e78b89cdcb3554a2 r-libarrow-linux-arm64-26.0.1.zip
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0d536d267eaa67446b0413b6c8e08a7945aad7165848fce6c6887f500a70e241058a109cdcb284d2e8b9f7df7aab420f468174645504e1db4602f2c99f1ea253 r-libarrow-linux-x86_64-26.0.1.zip
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
bd7d366de74419d6ea8a55b1e9432a59933bbb7656ac7d4a0b4ff2d99729424a1d66b71cc94532e403a97e79e2d0cb9c1d0e23fcd3b4cf26740d0bc31570492d r-libarrow-windows-x86_64-26.0.1.zip
Loading
Loading