Object Management

arrays

tutorials

python

Learn how to manage your array and group objects, including adding, modifying, and removing objects.

How to run this tutorial

You can run this tutorial in two ways:

Locally on your machine.
On TileDB Cloud.

However, since TileDB Cloud has a free tier, we strongly recommend that you sign up and run everything there, as that requires no installations or deployment.

This tutorial explains functionality around managing TileDB objects, mainly groups and arrays.

First, import the necessary libraries, set group and array URIs (that is, their paths, which in this tutorial will be on local storage), and delete any previously created groups and arrays with the same name.

Python
R

# Import necessary libraries
import os.path
import shutil

import numpy as np
import tiledb

# Set array URIs and names
array_uri = os.path.expanduser("~/object_mgmt_array")
array_name = "my_array"
new_array_uri = "my_array_new"

# Set group paths and names
grp_root_uri = os.path.expanduser("~/object_mgmt_grp_root")
grp1_uri = os.path.expanduser("~/object_mgmt_grp1")
grp2_uri = os.path.expanduser("~/object_mgmt_grp2")
grp3_uri = os.path.expanduser("~/object_mgmt_grp3")
grp1_name = "grp1"
grp2_name = "grp2"
grp3_name = "grp3"

# Delete all paths if they already exist
if os.path.exists(array_uri):
    shutil.rmtree(array_uri)
if os.path.exists(grp_root_uri):
    shutil.rmtree(grp_root_uri)
if os.path.exists(grp1_uri):
    shutil.rmtree(grp1_uri)
if os.path.exists(grp2_uri):
    shutil.rmtree(grp2_uri)
if os.path.exists(grp3_uri):
    shutil.rmtree(grp3_uri)

# Import necessary libraries
library(tiledb)

# Set array URIs and name
array_uri <- path.expand("~/object_mgmt_array_r")
array_name <- "my_array"
new_array_uri <- "my_array_new"

# Set group paths and names
grp_root_uri <- path.expand("~/object_mgmt_grp_root_r")
grp1_uri <- path.expand("~/object_mgmt_grp1_r")
grp2_uri <- path.expand("~/object_mgmt_grp2_r")
grp3_uri <- path.expand("~/object_mgmt_grp3_r")
grp1_name <- "grp1"
grp2_name <- "grp2"
grp3_name <- "grp3"

# Delete array if it already exists
if (file.exists(array_uri)) {
  unlink(array_uri, recursive = TRUE)
}
if (file.exists(grp_root_uri)) {
  unlink(grp_root_uri, recursive = TRUE)
}
if (file.exists(grp1_uri)) {
  unlink(grp1_uri, recursive = TRUE)
}
if (file.exists(grp2_uri)) {
  unlink(grp2_uri, recursive = TRUE)
}
if (file.exists(grp3_uri)) {
  unlink(grp3_uri, recursive = TRUE)
}

Next, create an array. In this example, the array is dense, but the functionality in this example applies to any array.

Python
R

# Create the two dimensions
d1 = tiledb.Dim(name="d1", domain=(1, 4), tile=2, dtype=np.int32)
d2 = tiledb.Dim(name="d2", domain=(1, 4), tile=2, dtype=np.int32)

# Create a domain using the two dimensions
dom = tiledb.Domain(d1, d2)
# Order of the dimensions matters when slicing subarrays.
# Remember to give priority to more selective dimensions to
# maximize the pruning power during slicing.

# Create an attribute
a = tiledb.Attr(name="a", dtype=np.int32)

# Create the array schema, setting `sparse=False` to indicate a dense array
sch = tiledb.ArraySchema(domain=dom, sparse=False, attrs=[a])

# Create the array on disk (it will initially be empty)
tiledb.Array.create(array_uri, sch)

# Create the two dimensions
d1 <- tiledb_dim("d1", c(1L, 4L), 2L, "INT32")
d2 <- tiledb_dim("d2", c(1L, 4L), 2L, "INT32")

# Create a domain using the two dimensions
dom <- tiledb_domain(dims = c(d1, d2))

# Create an attribute
a <- tiledb_attr("a", type = "INT32")

# Create the array schema, setting `sparse = FALSE` to indicate a dense array
sch <- tiledb_array_schema(dom, a, sparse = FALSE)

# Create the array on disk (it will initially be empty)
arr <- tiledb_array_create(array_uri, sch)

Create an object hierarchy, which sets the foundation for the rest of the examples.

Python
R

# Create groups
tiledb.Group.create(grp_root_uri)
tiledb.Group.create(grp1_uri)
tiledb.Group.create(grp2_uri)
tiledb.Group.create(grp3_uri)

# Create hierarchy
with tiledb.Group(grp_root_uri, "w") as grp_root:
    grp_root.add(grp1_uri, grp1_name)
    grp_root.add(grp2_uri, grp2_name)
    with tiledb.Group(grp1_uri, "w") as grp1:
        grp1.add(array_uri, array_name)
        grp1.add(grp3_uri, grp3_name)

# The group hierarchy looks as follows
with tiledb.Group(grp_root_uri, "r") as grp_root:
    print(grp_root)

object_mgmt_grp_root GROUP
|-- grp1 GROUP
|------ my_array ARRAY
|------ grp3 GROUP
|-- grp2 GROUP

# Create groups
tiledb_group_create(grp_root_uri)
tiledb_group_create(grp1_uri)
tiledb_group_create(grp2_uri)
tiledb_group_create(grp3_uri)

# Create hierarchy
grp_root <- tiledb_group(grp_root_uri, "WRITE")
tiledb_group_add_member(grp_root, grp1_uri, FALSE, grp1_name)
tiledb_group_add_member(grp_root, grp2_uri, FALSE, grp2_name)
grp_root <- tiledb_group_close(grp_root)

grp1 <- tiledb_group(grp1_uri, "WRITE")
tiledb_group_add_member(grp1, array_uri, FALSE, array_name)
tiledb_group_add_member(grp1, grp3_uri, FALSE, grp3_name)
grp1 <- tiledb_group_close(grp1)

grp_root <- tiledb_group_open(grp_root, "READ")
writeLines(tiledb_group_member_dump(grp_root, recursive = TRUE))

object_mgmt_grp_root_r GROUP
|-- grp1 GROUP
|------ my_array ARRAY
|------ grp3 GROUP
|-- grp2 GROUP

You can check the object type as follows.

Python
R

print(tiledb.object_type("invalid_path"))  # returns None
print(tiledb.object_type(grp1_uri))  # returns "group"
print(tiledb.object_type(array_uri))  # returns "array"

None
group
array

print(tiledb_object_type("invalid_path")) # "INVALID"
print(tiledb_object_type(grp1_uri)) # "GROUP"
print(tiledb_object_type(array_uri)) # "ARRAY"

[1] "INVALID"
[1] "GROUP"
[1] "ARRAY"

You can move or rename an object:

Python
R

# Rename array
tiledb.move(array_uri, new_array_uri)
print(tiledb.array_exists(array_uri))  # False
print(tiledb.array_exists(new_array_uri))  # True

# Rename array to original
tiledb.move(new_array_uri, array_uri)
print(tiledb.array_exists(array_uri))  # True
print(tiledb.array_exists(new_array_uri))  # False

False
True
True
False

# Rename array
invisible(tiledb_object_mv(array_uri, new_array_uri))
print(tiledb_object_type(array_uri) == "ARRAY") # FALSE
print(tiledb_object_type(new_array_uri) == "ARRAY") # TRUE

# Rename array to original
invisible(tiledb_object_mv(new_array_uri, array_uri))
print(tiledb_object_type(array_uri) == "ARRAY") # TRUE
print(tiledb_object_type(new_array_uri) == "ARRAY") # FALSE

[1] FALSE
[1] TRUE
[1] TRUE
[1] FALSE

You can also remove an object.

Python
R

tiledb.remove(grp1_uri)

invisible(tiledb_object_rm(grp1_uri))

Removing a subgroup from the root group should produce the following hierarchy:

Python
R

# The group hierarchy looks as follows
# NOTE: Removing the physical path of an object does not
# automatically remove it from the group members
with tiledb.Group(grp_root_uri, "r") as grp_root:
    print(grp_root)

object_mgmt_grp_root GROUP
|-- grp1 GROUP (does not exist)
|-- grp2 GROUP

# The group hierarchy looks as follows
# NOTE: Removing the physical path of an object does not
# automatically remove it from the group members
writeLines(tiledb_group_member_dump(grp_root, recursive = TRUE))

object_mgmt_grp_root_r GROUP
|-- grp1 GROUP (does not exist)
|-- grp2 GROUP

Warning

Deleting the physical storage path of an object doesn’t automatically delete any member created for this storage path in another group.

To mitigate this issue, you need to manually remove the corresponding member from the group.

Python
R

# Manually remove the deleted member from the group
with tiledb.Group(grp_root_uri, "w") as grp_root:
    grp_root.remove(grp1_name)

# Manually remove the deleted member from the group
grp_root <- tiledb_group_close(grp_root)
grp_root <- tiledb_group_open(grp_root, "WRITE")
tiledb_group_remove_member(grp_root, grp1_uri)
grp_root <- tiledb_group_close(grp_root)

After the preceding operation, the group hierarchy looks clean.

Python
R

# Now the group contains only valid members
with tiledb.Group(grp_root_uri, "r") as grp_root:
    print(grp_root)

object_mgmt_grp_root GROUP
|-- grp2 GROUP

grp_root <- tiledb_group_open(grp_root, "READ")

# Now the group contains only valid members
writeLines(tiledb_group_member_dump(grp_root, recursive = TRUE))

# Close the group
grp_root <- tiledb_group_close(grp_root)

object_mgmt_grp_root_r GROUP
|-- grp2 GROUP

Always clean up in the end.

Python
R

# Delete all paths
if os.path.exists(array_uri):
    shutil.rmtree(array_uri)
if os.path.exists(new_array_uri):
    shutil.rmtree(new_array_uri)
if os.path.exists(grp_root_uri):
    shutil.rmtree(grp_root_uri)
if os.path.exists(grp1_uri):
    shutil.rmtree(grp1_uri)
if os.path.exists(grp2_uri):
    shutil.rmtree(grp2_uri)
if os.path.exists(grp3_uri):
    shutil.rmtree(grp3_uri)

if (file.exists(array_uri)) {
  unlink(array_uri, recursive = TRUE)
}
if (file.exists(grp_root_uri)) {
  unlink(grp_root_uri, recursive = TRUE)
}
if (file.exists(grp1_uri)) {
  unlink(grp1_uri, recursive = TRUE)
}
if (file.exists(grp2_uri)) {
  unlink(grp2_uri, recursive = TRUE)
}
if (file.exists(grp3_uri)) {
  unlink(grp3_uri, recursive = TRUE)
}