Forum: help

RE: Help with sprites [ Reply ]
By: Yohann Demont on 2020-12-07 17:50

I dove deeper in the javascript and webgl.
Obviously this is a very though task, not sure I am sufficiently skilled.
But I can at least report some little things.

I am receiving a warning in js console:
WARNING: texture bound to texture unit 0 is not renderable. It maybe non-power-of-2 and have incompatible texture filtering.
This warning is repeated:
6 times if I want to draw 3 sprites with texture
10 times if I want to draw 4 sprites with texture
15 times if I want to draw 5 sprites with texture
55 times if I want to draw 10 sprites with texture
Despite that I am using power of 2 texture images (64 x 64, in my app) or (256 x 256) from rgl package 'nightfire.png', 'sunsleep.png', 'rgl2.png' and also set the canvas to 512 x 512 (through width and height parameters of rglwidgetOutput)

This warning only appears in Rstudio viewer window and when I use Mozilla Firefox
With these 2 browsers, interaction with the scene is almost totaly blocked until everything is rendered.

However, it does not print with Google Chrome nor Microsoft Edge.
With the 2 latters, I can interact with the scene meanwhile textures are displayed one by one which is much more confortable

Also, but this is not related with sprites, rglClass.min.js leaves some variables as global z, w, i and last.
z, w, i seem to com from getPieces() and last from initObj()

rglwidgetClass.prototype.getPieces = function (context, objid, subid, obj) {
var depth,
z, w, i, // having them here remove them from global
n = obj.centers.length,

rglwidgetClass.prototype.initObj = function (id) {
...
var nc,
cofs,
nofs,
radofs,
oofs,
tofs,
vnew,
fnew,
alias,
colors,
key,
last, // having it here removes it from global
selection,

RE: Help with sprites [ Reply ]
By: Yohann Demont on 2020-12-05 16:20

[forum:48576]

Results from some of the explorations I did are presented in the code under
- one solution for passing data uri.
This solution is 'easy' in the sense that I don't need to change things in rgl code.

- the other one is to reduce the impact on the view due to the addition of sprites to the scene.
However, the bounding box is still stretched and we need to recreate axes.

I will take a deeper look at webgl and javascript.
Because the timing for rglwidget() to create the object in R looks very fast,
however the rendering by the browser seems to be the bottleneck.
(For example, if we set N to 500 instead of 50 in the code below, we see sprites getting ready one by one very slowly).
Maybe, I will try also to remove lights, antialising.

require(rgl, quietly = TRUE, warn.conflicts = FALSE)
require(shiny, quietly = TRUE, warn.conflicts = FALSE)

N = 50
dat = iris[sample(1:nrow(iris), size = N, replace = TRUE), ]
dat[, 1] = dat[, 1] / 10
dat[, 2] = dat[, 2] * 100
textures = list.files(path = system.file(package = "rgl", "textures"), all.files = FALSE, include.dirs = FALSE, full.names = TRUE)
textures = textures[-1]

# here knitr is used to convert texture to data uri
require(knitr, quietly = TRUE, warn.conflicts = FALSE)
str(system.time({uri = sapply(textures, knitr::image_uri)}))
uri = sample(uri, size = nrow(dat), replace = TRUE)
textures = sample(textures, size = nrow(dat), replace = TRUE)

# function to expand limits
expand_lim = function(x, expand = 1.2) return(x + (expand - 1) * diff(x) / 2 * c(-1,1))
xlim = expand_lim(range(dat[, 1]))
ylim = expand_lim(range(dat[, 2]))
zlim = expand_lim(range(dat[, 3]))

ui <- fluidPage(
sidebarLayout(
mainPanel(tags$div(id = "plot_3D_loc",
verbatimTextOutput(outputId = "time2"),
verbatimTextOutput(outputId = "time3"),
rglwidgetOutput("plot_3D_1"),
rglwidgetOutput("plot_3D_2"),
rglwidgetOutput("plot_3D_3"))),
sidebarPanel(radioButtons(inputId = "set_observer", label = "set observer", choices = c("TRUE", "FALSE"), selected = "TRUE"),
radioButtons(inputId = "add_bbox", label = "add bbox", choices = c("TRUE", "FALSE"), selected = "FALSE"))
))

server <- function(input, output, session) {
while(rgl.cur()!=0) try(rgl.close(), silent = TRUE)
dev1 = open3d(useNULL = TRUE)
dev2 = open3d(useNULL = TRUE)
dev3 = open3d(useNULL = TRUE)

output$plot_3D_1 <- renderRglwidget({
rgl.set(dev1)
clear3d()
plot3d(x = dat[, 1:3], type = "s", size = 1, col = as.integer(dat[, "Species"]), aspect = TRUE)
rglwidget()
})

# this one shows how to adjust viewer location so as to see the sprites
output$plot_3D_2 <- renderRglwidget({
rgl.set(dev2)
clear3d()
points3d(x=xlim, y=ylim, z=zlim, size=0)
aspect3d(1,1,1)

# observer3d is called a 1st time with param auto = TRUE to compute viewer location
observer3d(x = 0, y = 0, z = 1, auto = TRUE)
# then it is called once again because it return the previous value
prev_obs <- observer3d(x = 0, y = 0, z = 1, auto = TRUE)

ids <- lapply(1:nrow(dat), FUN = function(i_row) {
sprites3d(dat[i_row, 1:3], radius = 10, alpha = 0.6,
col = as.integer(dat[i_row, "Species"]),
fixedSize = FALSE, lit = FALSE,
textype = "rgb",
texture = textures[i_row])
})

# finally observer3d is called with argument auto = FALSE and with the previous viewer location values
if(input$set_observer == "TRUE") observer3d(x = prev_obs[1], y = prev_obs[2], z = prev_obs[3], auto = FALSE)
# however bounding box is still wrong so we have to create the decoration manually
# I am not showing how to do it here but I think it could be helpful to read the article at: (I have no affiliation with it)
# http://www.sthda.com/english/wiki/a-complete-guide-to-3d-visualization-device-system-in-r-r-software-and-data-visualization
if(input$add_bbox == "TRUE") bbox3d()
rglwidget()
})

# this one is for demonstration of base64 data uri injection
output$plot_3D_3 <- renderRglwidget({
rgl.set(dev3)
clear3d()
points3d(x=xlim, y=ylim, z=zlim, size=0)
aspect3d(1,1,1)
observer3d(x = 0, y = 0, z = 1, auto = TRUE)
prev_obs <- observer3d(x = 0, y = 0, z = 1, auto = TRUE)
ids <- lapply(1:nrow(dat), FUN = function(i_row) {
sprites3d(dat[i_row, 1:3], radius = 10, alpha = 0.6,
col = as.integer(dat[i_row, "Species"]),
fixedSize = FALSE, lit = FALSE,
textype = "rgb",
texture = NULL)
})
observer3d(x = prev_obs[1], y = prev_obs[2], z = prev_obs[3], auto = FALSE)

# here we use rglwidget to build the object but don't return it directly
pre <- rglwidget()

# inject data uri
for(i in 1:nrow(dat)) {
# It seems it is where to place data uri
pre$x$objects[[as.character(ids[i])]]$material$uri <- unname(uri[i])

# By comparison I saw that emtpy texture is 32824
# Although I am not completely sure that 32828 will always work ?
pre$x$objects[[as.character(ids[i])]]$flags <- 32828
}
return(pre)
})
}

shinyApp(ui, server)

# on my computer I can gain one third by injecting directly the data uri.
# obviously this very specific to my needs since I already have the images as data uri

RE: Help with sprites [ Reply ]
By: Duncan Murdoch on 2020-12-04 09:15

[forum:48516]

Currently the rgl:::convertScene function takes the material texture element and calls knitr::image_uri to encode it. I imagine it would be a relatively easy change to recognize if the texture had been specified as a URI already and skip the encoding.

However, I doubt if this is going to make a huge difference to your timing. Shiny is inherently slow when dealing with rgl: typically the data for an rgl scene is really large (it contains all the textures, point coordinates, etc), and Shiny downloads a complete copy every time you make any change. If you need speed, try to make all the changes in Javascript, using the functions described in the WebGL vignette (or using raw Javascript).

Help with sprites [ Reply ]
By: Yohann Demont on 2020-12-03 08:47

[forum:48499]

Hi,

I would like to use sprites in a shiny app.
After several tries, I have some questions I did not managed to solve.
- Is there a simple way (without forking) to directly feed texture argument in rgl.sprites(), sprites3d(), rgl.material() or other, with base64 data/uri (data:image/png;base64,iV and so on ...).
In my app, I already have images as data uri and writing them back to files so as to pass file path location to texture argument so that rgl can read them and convert them back to data uri is highly inefficient in my case.
If there is no easy way to do it, I would be happy if you can provide my an entry point to do such thing (eg I feed in R with a very small png and then I replace all generated data uri by the ones I already have)
- It appears that having a lots of sprites in the shiny app requires some time for the widget to be ready (for example before being able to rotate). It looks that images are first displayed and secondly transparency is applied and then user can handle the widget.
What would be the best settings to keep alpha=0.6, with texttype="rgb" but reducing the time for the 2nd step .
I there a way to monitor that the widget is computing something and not yet ready for interaction. A progress bar would be fantastic, but a flag would be enough to disable all controls until the widget gets ready.
In the same vein, could the rendering be cancellable ?
- Finally (this also apply to wgl and not only to shiny), it seems that injecting sprites in the graph changes the bounding box calculation. It stretches the limits largely outside of the actual points coordinates.

Here is an example illustrating question 3:

require(rgl, quietly = TRUE, warn.conflicts = FALSE)
require(shiny, quietly = TRUE, warn.conflicts = FALSE)

dat = iris[sample(1:nrow(iris), size = 50, replace = FALSE), ]
dat[, 1] = dat[, 1] / 10
dat[, 2] = dat[, 2] * 100
textures = list.files(path = system.file(package = "rgl", "textures"), all.files = FALSE, include.dirs = FALSE, full.names = TRUE)
textures = textures[-1]
textures = sample(textures, size = nrow(dat), replace = TRUE)
xlim = range(dat[, 1])
ylim = range(dat[, 2])
zlim = range(dat[, 3])

ui <- fluidPage(
sidebarLayout(
mainPanel(tags$div(id = "plot_3D_loc",
rglwidgetOutput("plot_3D_1"),
rglwidgetOutput("plot_3D_2"),
rglwidgetOutput("plot_3D_3"),
rglwidgetOutput("plot_3D_4"))),
sidebarPanel(radioButtons(inputId = "set_aspect2", label = "set aspect sprites", choices = c("TRUE", "FALSE"), selected = "TRUE"),
downloadButton(outputId = "save", label = "Save"))
))

server <- function(input, output, session) {
redraw = reactiveValues(observer = numeric())
open3d(useNULL = TRUE)

# here is the aspect I would like to get but with sprites
output$plot_3D_1 <- renderRglwidget({
clear3d()
plot3d(x = dat[, 1:3], type = "s", size = 1, col = as.integer(dat[, "Species"]), aspect = TRUE)
rglwidget()
})

# this one with sprites gets an anormal aspect,
# although images are well positionned (i.e. at their good x,y,z coordinates)
# the bounding box is completly wrong
# and use of aspect(1,1,1) does not solve the problem
output$plot_3D_2 <- renderRglwidget({
clear3d()
# as far as I understood it is not possible to draw several sprites with several different textures
# i.e. texture argument is scalar so I use an lapply do draw every single sprite
lapply(1:length(textures), FUN = function(i_file) {
sprites3d(dat[i_file, ], radius = 10,
col = "grey", fixedSize = FALSE, lit = FALSE,
alpha = 0.6, textype = "rgb",
texture = textures[i_file])
})
decorate3d(xlim = xlim, xlab = "x", ylim = ylim, ylab = "y", zlim = zlim, zlab = "z")
if(input$set_aspect2 == "TRUE") aspect3d(1,1,1)
rglwidget()
})

# if we don't use plot3d but sphere3d and then create the axes/box it is ok
output$plot_3D_3 <- renderRglwidget({
clear3d()
spheres3d(x = dat[, 1:3], radius = 1, col = as.integer(dat[, "Species"]))
aspect3d(1,1,1)
decorate3d(xlim = xlim, xlab = "x", ylim = ylim, ylab = "y", zlim = zlim, zlab = "z")
rglwidget()
})

# if we draw each individual sphere3d it is still ok
output$plot_3D_4 <- renderRglwidget({
clear3d()
lapply(1:nrow(dat), FUN = function(i_row) {
spheres3d(x = dat[i_row, 1:3], radius = 1, col = as.integer(dat[i_row, "Species"]))
})
aspect3d(1,1,1)
decorate3d(xlim = xlim, xlab = "x", ylim = ylim, ylab = "y", zlim = zlim, zlab = "z")
rglwidget()
})
}

shinyApp(ui, server)

Best,
Yohann